From 77a241debfbd4fa70b97047635de86ba0a374497 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Tue, 20 Oct 2020 19:10:52 +0200 Subject: [PATCH 001/254] Adjust jcheck config to allow commits without issue ID and merge commits --- .jcheck/conf | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.jcheck/conf b/.jcheck/conf index 40d53c860d59f..915aa07e9c700 100644 --- a/.jcheck/conf +++ b/.jcheck/conf @@ -3,7 +3,7 @@ project=jdk jbs=JDK [checks] -error=author,committer,reviewers,merge,issues,executable,symlink,message,hg-tag,whitespace,problemlists +error=author,committer,reviewers,executable,symlink,message,hg-tag,whitespace,problemlists [repository] tags=(?:jdk-(?:[1-9]([0-9]*)(?:\.(?:0|[1-9][0-9]*)){0,4})(?:\+(?:(?:[0-9]+))|(?:-ga)))|(?:jdk[4-9](?:u\d{1,3})?-(?:(?:b\d{2,3})|(?:ga)))|(?:hs\d\d(?:\.\d{1,2})?-b\d\d) @@ -16,9 +16,6 @@ domain=openjdk.org [checks "whitespace"] files=.*\.cpp|.*\.hpp|.*\.c|.*\.h|.*\.java|.*\.cc|.*\.hh|.*\.m|.*\.mm -[checks "merge"] -message=Merge - [checks "reviewers"] reviewers=1 ignore=duke @@ -26,8 +23,5 @@ ignore=duke [checks "committer"] role=committer -[checks "issues"] -pattern=^([124-8][0-9]{6}): (\S.*)$ - [checks "problemlists"] dirs=test/jdk|test/langtools|test/lib-test|test/hotspot/jtreg|test/jaxp From 5077810c1087fb19cfdb4eded207ba8e2e01170f Mon Sep 17 00:00:00 2001 From: Bernd Mathiske Date: Tue, 29 Sep 2020 22:50:48 +0000 Subject: [PATCH 002/254] Add 'generational' as value choice for JVM option 'ShenandoahGCMode'. --- .../mode/shenandoahGenerationalMode.cpp | 67 +++++++++++++++++++ .../mode/shenandoahGenerationalMode.hpp | 42 ++++++++++++ .../gc/shenandoah/mode/shenandoahMode.hpp | 1 + .../share/gc/shenandoah/shenandoahHeap.cpp | 3 + .../gc/shenandoah/shenandoah_globals.hpp | 3 +- 5 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp create mode 100644 src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp new file mode 100644 index 0000000000000..e3c770e4afc2f --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2020, Red Hat, Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shenandoah/shenandoahConcurrentRoots.hpp" +#include "gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp" +#include "gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.hpp" +#include "gc/shenandoah/heuristics/shenandoahCompactHeuristics.hpp" +#include "gc/shenandoah/heuristics/shenandoahStaticHeuristics.hpp" +#include "gc/shenandoah/mode/shenandoahGenerationalMode.hpp" +#include "runtime/java.hpp" + +void ShenandoahGenerationalMode::initialize_flags() const { + if (ShenandoahConcurrentRoots::can_do_concurrent_class_unloading()) { + FLAG_SET_DEFAULT(ShenandoahSuspendibleWorkers, true); + FLAG_SET_DEFAULT(VerifyBeforeExit, false); + } + + SHENANDOAH_ERGO_ENABLE_FLAG(ExplicitGCInvokesConcurrent); + SHENANDOAH_ERGO_ENABLE_FLAG(ShenandoahImplicitGCInvokesConcurrent); + + // Final configuration checks + SHENANDOAH_CHECK_FLAG_SET(ShenandoahLoadRefBarrier); + SHENANDOAH_CHECK_FLAG_UNSET(ShenandoahStoreValEnqueueBarrier); + SHENANDOAH_CHECK_FLAG_SET(ShenandoahSATBBarrier); + SHENANDOAH_CHECK_FLAG_SET(ShenandoahCASBarrier); + SHENANDOAH_CHECK_FLAG_SET(ShenandoahCloneBarrier); +} + +ShenandoahHeuristics* ShenandoahGenerationalMode::initialize_heuristics() const { + if (ShenandoahGCHeuristics != NULL) { + if (strcmp(ShenandoahGCHeuristics, "aggressive") == 0) { + return new ShenandoahAggressiveHeuristics(); + } else if (strcmp(ShenandoahGCHeuristics, "static") == 0) { + return new ShenandoahStaticHeuristics(); + } else if (strcmp(ShenandoahGCHeuristics, "adaptive") == 0) { + return new ShenandoahAdaptiveHeuristics(); + } else if (strcmp(ShenandoahGCHeuristics, "compact") == 0) { + return new ShenandoahCompactHeuristics(); + } else { + vm_exit_during_initialization("Unknown -XX:ShenandoahGCHeuristics option"); + } + } + ShouldNotReachHere(); + return NULL; +} diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp new file mode 100644 index 0000000000000..6d7482d0bac11 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2020, Red Hat, Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHENANDOAH_MODE_SHENANDOAHGENERATIONALMODE_HPP +#define SHARE_GC_SHENANDOAH_MODE_SHENANDOAHGENERATIONALMODE_HPP + +#include "gc/shenandoah/mode/shenandoahMode.hpp" + +class ShenandoahHeuristics; + +class ShenandoahGenerationalMode : public ShenandoahMode { +public: + virtual void initialize_flags() const; + virtual ShenandoahHeuristics* initialize_heuristics() const; + virtual const char* name() { return "Generational"; } + virtual bool is_diagnostic() { return false; } + virtual bool is_experimental() { return false; } + virtual bool is_generational() { return true; } +}; + +#endif // SHARE_GC_SHENANDOAH_MODE_SHENANDOAHGENERATIONALMODE_HPP diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.hpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.hpp index 5af6fa826d51c..1da6264e19459 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.hpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.hpp @@ -52,6 +52,7 @@ class ShenandoahMode : public CHeapObj { virtual const char* name() = 0; virtual bool is_diagnostic() = 0; virtual bool is_experimental() = 0; + virtual bool is_generational() { return false; } }; #endif // SHARE_GC_SHENANDOAH_MODE_SHENANDOAHMODE_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 5b45c0930ac9e..954661e442e71 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -64,6 +64,7 @@ #include "gc/shenandoah/shenandoahVMOperations.hpp" #include "gc/shenandoah/shenandoahWorkGroup.hpp" #include "gc/shenandoah/shenandoahWorkerPolicy.hpp" +#include "gc/shenandoah/mode/shenandoahGenerationalMode.hpp" #include "gc/shenandoah/mode/shenandoahIUMode.hpp" #include "gc/shenandoah/mode/shenandoahPassiveMode.hpp" #include "gc/shenandoah/mode/shenandoahSATBMode.hpp" @@ -409,6 +410,8 @@ void ShenandoahHeap::initialize_heuristics() { _gc_mode = new ShenandoahIUMode(); } else if (strcmp(ShenandoahGCMode, "passive") == 0) { _gc_mode = new ShenandoahPassiveMode(); + } else if (strcmp(ShenandoahGCMode, "generational") == 0) { + _gc_mode = new ShenandoahGenerationalMode(); } else { vm_exit_during_initialization("Unknown -XX:ShenandoahGCMode option"); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp index 5417cb5a9fcd1..505b4a7c2a40a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp @@ -62,7 +62,8 @@ "barriers are in in use. Possible values are:" \ " satb - snapshot-at-the-beginning concurrent GC (three pass mark-evac-update);" \ " iu - incremental-update concurrent GC (three pass mark-evac-update);" \ - " passive - stop the world GC only (either degenerated or full)") \ + " passive - stop the world GC only (either degenerated or full);" \ + " generational - generational concurrent GC") \ \ product(ccstr, ShenandoahGCHeuristics, "adaptive", \ "GC heuristics to use. This fine-tunes the GC mode selected, " \ From 73a638815ed79fed0bd8a18145b534e89227d677 Mon Sep 17 00:00:00 2001 From: Bernd Mathiske Date: Mon, 16 Nov 2020 13:32:16 +0000 Subject: [PATCH 003/254] Card marking write barrier for generational mode. Reviewed-by: rkennke --- .../c1/shenandoahBarrierSetC1_x86.cpp | 12 +- .../shenandoahBarrierSetAssembler_x86.cpp | 109 ++++++++++++- .../shenandoahBarrierSetAssembler_x86.hpp | 7 +- .../shenandoah/c1/shenandoahBarrierSetC1.cpp | 78 ++++++++++ .../shenandoah/c1/shenandoahBarrierSetC1.hpp | 2 + .../shenandoah/c2/shenandoahBarrierSetC2.cpp | 144 +++++++++++++++++- .../shenandoah/c2/shenandoahBarrierSetC2.hpp | 14 +- .../gc/shenandoah/shenandoahBarrierSet.cpp | 26 +++- .../gc/shenandoah/shenandoahBarrierSet.hpp | 12 +- .../shenandoahBarrierSet.inline.hpp | 35 ++++- .../gc/shenandoah/shenandoahCardTable.cpp | 31 ++++ .../gc/shenandoah/shenandoahCardTable.hpp | 47 ++++++ .../share/gc/shenandoah/shenandoahHeap.cpp | 44 +++--- 13 files changed, 519 insertions(+), 42 deletions(-) create mode 100644 src/hotspot/share/gc/shenandoah/shenandoahCardTable.cpp create mode 100644 src/hotspot/share/gc/shenandoah/shenandoahCardTable.hpp diff --git a/src/hotspot/cpu/x86/gc/shenandoah/c1/shenandoahBarrierSetC1_x86.cpp b/src/hotspot/cpu/x86/gc/shenandoah/c1/shenandoahBarrierSetC1_x86.cpp index 58dcd9ed5fb86..35412bf8871f3 100644 --- a/src/hotspot/cpu/x86/gc/shenandoah/c1/shenandoahBarrierSetC1_x86.cpp +++ b/src/hotspot/cpu/x86/gc/shenandoah/c1/shenandoahBarrierSetC1_x86.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved. + * Copyright (c) 2018, 2020 Red Hat, Inc. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -89,7 +89,14 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess& access, LI return result; } } - return BarrierSetC1::atomic_cmpxchg_at_resolved(access, cmp_value, new_value); + + LIR_Opr result = BarrierSetC1::atomic_cmpxchg_at_resolved(access, cmp_value, new_value); + + if (access.is_oop()) { + post_barrier(access, access.resolved_addr(), new_value.result()); + } + + return result; } LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess& access, LIRItem& value) { @@ -119,6 +126,7 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess& access, LIRIt pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), LIR_OprFact::illegalOpr, result /* pre_val */); } + post_barrier(access, access.resolved_addr(), result); } return result; diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp index 2d9d1c5215d85..5195a527953f8 100644 --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp @@ -31,6 +31,7 @@ #include "gc/shenandoah/shenandoahRuntime.hpp" #include "gc/shenandoah/shenandoahThreadLocalData.hpp" #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" +#include "gc/shenandoah/mode/shenandoahMode.hpp" #include "interpreter/interpreter.hpp" #include "interpreter/interp_masm.hpp" #include "runtime/sharedRuntime.hpp" @@ -565,8 +566,57 @@ void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet d } } +void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj, Address dst) { + if (!ShenandoahHeap::heap()->mode()->is_generational()) { + return; + } + + // Does a store check for the oop in register obj. The content of + // register obj is destroyed afterwards. + BarrierSet* bs = BarrierSet::barrier_set(); + + ShenandoahBarrierSet* ctbs = barrier_set_cast(bs); + CardTable* ct = ctbs->card_table(); + + __ shrptr(obj, CardTable::card_shift); + + Address card_addr; + + // The calculation for byte_map_base is as follows: + // byte_map_base = _byte_map - (uintptr_t(low_bound) >> card_shift); + // So this essentially converts an address to a displacement and it will + // never need to be relocated. On 64bit however the value may be too + // large for a 32bit displacement. + intptr_t byte_map_base = (intptr_t)ct->byte_map_base(); + if (__ is_simm32(byte_map_base)) { + card_addr = Address(noreg, obj, Address::times_1, byte_map_base); + } else { + // By doing it as an ExternalAddress 'byte_map_base' could be converted to a rip-relative + // displacement and done in a single instruction given favorable mapping and a + // smarter version of as_Address. However, 'ExternalAddress' generates a relocation + // entry and that entry is not properly handled by the relocation code. + AddressLiteral cardtable((address)byte_map_base, relocInfo::none); + Address index(noreg, obj, Address::times_1); + card_addr = __ as_Address(ArrayAddress(cardtable, index)); + } + + int dirty = CardTable::dirty_card_val(); + if (UseCondCardMark) { + Label L_already_dirty; + if (ct->scanned_concurrently()) { + __ membar(Assembler::StoreLoad); + } + __ cmpb(card_addr, dirty); + __ jcc(Assembler::equal, L_already_dirty); + __ movb(card_addr, dirty); + __ bind(L_already_dirty); + } else { + __ movb(card_addr, dirty); + } +} + void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, - Address dst, Register val, Register tmp1, Register tmp2) { + Address dst, Register val, Register tmp1, Register tmp2) { bool on_oop = is_reference_type(type); bool in_heap = (decorators & IN_HEAP) != 0; @@ -608,6 +658,7 @@ void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet } else { storeval_barrier(masm, val, tmp3); BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg); + store_check(masm, tmp1, dst); } NOT_LP64(imasm->restore_bcp()); } else { @@ -799,6 +850,62 @@ void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, } } +#ifdef PRODUCT +#define BLOCK_COMMENT(str) /* nothing */ +#else +#define BLOCK_COMMENT(str) __ block_comment(str) +#endif + +#define BIND(label) bind(label); BLOCK_COMMENT(#label ":") + +#define TIMES_OOP (UseCompressedOops ? Address::times_4 : Address::times_8) + +void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, Register addr, Register count, Register tmp) { + if (!ShenandoahHeap::heap()->mode()->is_generational()) { + return; + } + + BarrierSet *bs = BarrierSet::barrier_set(); + ShenandoahBarrierSet* ctbs = barrier_set_cast(bs); + CardTable* ct = ctbs->card_table(); + intptr_t disp = (intptr_t) ct->byte_map_base(); + + Label L_loop, L_done; + const Register end = count; + assert_different_registers(addr, end); + + __ testl(count, count); + __ jcc(Assembler::zero, L_done); // zero count - nothing to do + + +#ifdef _LP64 + __ leaq(end, Address(addr, count, TIMES_OOP, 0)); // end == addr+count*oop_size + __ subptr(end, BytesPerHeapOop); // end - 1 to make inclusive + __ shrptr(addr, CardTable::card_shift); + __ shrptr(end, CardTable::card_shift); + __ subptr(end, addr); // end --> cards count + + __ mov64(tmp, disp); + __ addptr(addr, tmp); +__ BIND(L_loop); + __ movb(Address(addr, count, Address::times_1), 0); + __ decrement(count); + __ jcc(Assembler::greaterEqual, L_loop); +#else + __ lea(end, Address(addr, count, Address::times_ptr, -wordSize)); + __ shrptr(addr, CardTable::card_shift); + __ shrptr(end, CardTable::card_shift); + __ subptr(end, addr); // end --> count +__ BIND(L_loop); + Address cardtable(addr, count, Address::times_1, disp); + __ movb(cardtable, 0); + __ decrement(count); + __ jcc(Assembler::greaterEqual, L_loop); +#endif + +__ BIND(L_done); +} + #undef __ #ifdef COMPILER1 diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp index 60aa3b4600d44..e3011ddc77688 100644 --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved. + * Copyright (c) 2018, 2020, Red Hat, Inc. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -56,12 +56,16 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler { bool tosca_live, bool expand_call); + void store_check(MacroAssembler* masm, Register obj, Address dst); + void load_reference_barrier_not_null(MacroAssembler* masm, Register dst, Address src); void storeval_barrier_impl(MacroAssembler* masm, Register dst, Register tmp); address generate_shenandoah_lrb(StubCodeGenerator* cgen); + void gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, Register addr, Register count, Register tmp); + public: static address shenandoah_lrb(); @@ -88,7 +92,6 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler { virtual void try_resolve_jobject_in_native(MacroAssembler* masm, Register jni_env, Register obj, Register tmp, Label& slowpath); virtual void barrier_stubs_init(); - }; #endif // CPU_X86_GC_SHENANDOAH_SHENANDOAHBARRIERSETASSEMBLER_X86_HPP diff --git a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp index 9754f495034b8..9198524a90d8e 100644 --- a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp +++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp @@ -25,6 +25,7 @@ #include "precompiled.hpp" #include "c1/c1_IR.hpp" #include "gc/shared/satbMarkQueue.hpp" +#include "gc/shenandoah/mode/shenandoahMode.hpp" #include "gc/shenandoah/shenandoahBarrierSet.hpp" #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" @@ -186,6 +187,16 @@ void ShenandoahBarrierSetC1::store_at_resolved(LIRAccess& access, LIR_Opr value) value = storeval_barrier(access.gen(), value, access.access_emit_info(), access.decorators()); } BarrierSetC1::store_at_resolved(access, value); + + if (access.is_oop()) { + DecoratorSet decorators = access.decorators(); + bool is_array = (decorators & IS_ARRAY) != 0; + bool on_anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0; + + bool precise = is_array || on_anonymous; + LIR_Opr post_addr = precise ? access.resolved_addr() : access.base().opr(); + post_barrier(access, post_addr, value); + } } LIR_Opr ShenandoahBarrierSetC1::resolve_address(LIRAccess& access, bool resolve_in_register) { @@ -280,3 +291,70 @@ void ShenandoahBarrierSetC1::generate_c1_runtime_stubs(BufferBlob* buffer_blob) false, &lrb_native_code_gen_cl); } } + +void ShenandoahBarrierSetC1::post_barrier(LIRAccess& access, LIR_OprDesc* addr, LIR_OprDesc* new_val) { + if (!ShenandoahHeap::heap()->mode()->is_generational()) { + return; + } + + DecoratorSet decorators = access.decorators(); + LIRGenerator* gen = access.gen(); + bool in_heap = (decorators & IN_HEAP) != 0; + if (!in_heap) { + return; + } + + BarrierSet* bs = BarrierSet::barrier_set(); + ShenandoahBarrierSet* ctbs = barrier_set_cast(bs); + CardTable* ct = ctbs->card_table(); + LIR_Const* card_table_base = new LIR_Const(ct->byte_map_base()); + if (addr->is_address()) { + LIR_Address* address = addr->as_address_ptr(); + // ptr cannot be an object because we use this barrier for array card marks + // and addr can point in the middle of an array. + LIR_Opr ptr = gen->new_pointer_register(); + if (!address->index()->is_valid() && address->disp() == 0) { + __ move(address->base(), ptr); + } else { + assert(address->disp() != max_jint, "lea doesn't support patched addresses!"); + __ leal(addr, ptr); + } + addr = ptr; + } + assert(addr->is_register(), "must be a register at this point"); + + LIR_Opr tmp = gen->new_pointer_register(); + if (TwoOperandLIRForm) { + __ move(addr, tmp); + __ unsigned_shift_right(tmp, CardTable::card_shift, tmp); + } else { + __ unsigned_shift_right(addr, CardTable::card_shift, tmp); + } + + LIR_Address* card_addr; + if (gen->can_inline_as_constant(card_table_base)) { + card_addr = new LIR_Address(tmp, card_table_base->as_jint(), T_BYTE); + } else { + card_addr = new LIR_Address(tmp, gen->load_constant(card_table_base), T_BYTE); + } + + LIR_Opr dirty = LIR_OprFact::intConst(CardTable::dirty_card_val()); + if (UseCondCardMark) { + LIR_Opr cur_value = gen->new_register(T_INT); + if (ct->scanned_concurrently()) { + __ membar_storeload(); + } + __ move(card_addr, cur_value); + + LabelObj* L_already_dirty = new LabelObj(); + __ cmp(lir_cond_equal, cur_value, dirty); + __ branch(lir_cond_equal, L_already_dirty->label()); + __ move(dirty, card_addr); + __ branch_destination(L_already_dirty->label()); + } else { + if (ct->scanned_concurrently()) { + __ membar_storestore(); + } + __ move(dirty, card_addr); + } +} diff --git a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp index e97f3af0926c1..b619fc5954911 100644 --- a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp +++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp @@ -229,6 +229,8 @@ class ShenandoahBarrierSetC1 : public BarrierSetC1 { virtual LIR_Opr atomic_xchg_at_resolved(LIRAccess& access, LIRItem& value); + void post_barrier(LIRAccess& access, LIR_OprDesc* addr, LIR_OprDesc* new_val); + public: virtual void generate_c1_runtime_stubs(BufferBlob* buffer_blob); diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp index 19b01ddfd5854..df29753c0d52f 100644 --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved. + * Copyright (c) 2018, 2020, Red Hat, Inc. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -32,6 +32,7 @@ #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp" #include "gc/shenandoah/c2/shenandoahSupport.hpp" #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" +#include "gc/shenandoah/mode/shenandoahMode.hpp" #include "opto/arraycopynode.hpp" #include "opto/escape.hpp" #include "opto/graphKit.hpp" @@ -452,6 +453,107 @@ void ShenandoahBarrierSetC2::insert_pre_barrier(GraphKit* kit, Node* base_oop, N kit->final_sync(ideal); } +Node* ShenandoahBarrierSetC2::byte_map_base_node(GraphKit* kit) const { + BarrierSet* bs = BarrierSet::barrier_set(); + ShenandoahBarrierSet* ctbs = barrier_set_cast(bs); + CardTable::CardValue* card_table_base = ctbs->card_table()->byte_map_base(); + if (card_table_base != NULL) { + return kit->makecon(TypeRawPtr::make((address)card_table_base)); + } else { + return kit->null(); + } +} + +void ShenandoahBarrierSetC2::post_barrier(GraphKit* kit, + Node* ctl, + Node* oop_store, + Node* obj, + Node* adr, + uint adr_idx, + Node* val, + BasicType bt, + bool use_precise) const { + if (!ShenandoahHeap::heap()->mode()->is_generational()) { + return; + } + + ShenandoahBarrierSet* ctbs = barrier_set_cast(BarrierSet::barrier_set()); + CardTable* ct = ctbs->card_table(); + // No store check needed if we're storing a NULL or an old object + // (latter case is probably a string constant). The concurrent + // mark sweep garbage collector, however, needs to have all nonNull + // oop updates flagged via card-marks. + if (val != NULL && val->is_Con()) { + // must be either an oop or NULL + const Type* t = val->bottom_type(); + if (t == TypePtr::NULL_PTR || t == Type::TOP) + // stores of null never (?) need barriers + return; + } + + if (ReduceInitialCardMarks && obj == kit->just_allocated_object(kit->control())) { + // We can skip marks on a freshly-allocated object in Eden. + // Keep this code in sync with new_deferred_store_barrier() in runtime.cpp. + // That routine informs GC to take appropriate compensating steps, + // upon a slow-path allocation, so as to make this card-mark + // elision safe. + return; + } + + if (!use_precise) { + // All card marks for a (non-array) instance are in one place: + adr = obj; + } + // (Else it's an array (or unknown), and we want more precise card marks.) + assert(adr != NULL, ""); + + IdealKit ideal(kit, true); + + // Convert the pointer to an int prior to doing math on it + Node* cast = __ CastPX(__ ctrl(), adr); + + // Divide by card size + Node* card_offset = __ URShiftX( cast, __ ConI(CardTable::card_shift) ); + + // Combine card table base and card offset + Node* card_adr = __ AddP(__ top(), byte_map_base_node(kit), card_offset ); + + // Get the alias_index for raw card-mark memory + int adr_type = Compile::AliasIdxRaw; + Node* zero = __ ConI(0); // Dirty card value + + if (UseCondCardMark) { + if (ct->scanned_concurrently()) { + kit->insert_mem_bar(Op_MemBarVolatile, oop_store); + __ sync_kit(kit); + } + // The classic GC reference write barrier is typically implemented + // as a store into the global card mark table. Unfortunately + // unconditional stores can result in false sharing and excessive + // coherence traffic as well as false transactional aborts. + // UseCondCardMark enables MP "polite" conditional card mark + // stores. In theory we could relax the load from ctrl() to + // no_ctrl, but that doesn't buy much latitude. + Node* card_val = __ load( __ ctrl(), card_adr, TypeInt::BYTE, T_BYTE, adr_type); + __ if_then(card_val, BoolTest::ne, zero); + } + + // Smash zero into card + if(!ct->scanned_concurrently()) { + __ store(__ ctrl(), card_adr, zero, T_BYTE, adr_type, MemNode::unordered); + } else { + // Specialized path for CM store barrier + __ storeCM(__ ctrl(), card_adr, zero, oop_store, adr_idx, T_BYTE, adr_type); + } + + if (UseCondCardMark) { + __ end_if(); + } + + // Final sync IdealKit and GraphKit. + kit->final_sync(ideal); +} + #undef __ const TypeFunc* ShenandoahBarrierSetC2::write_ref_field_pre_entry_Type() { @@ -518,6 +620,12 @@ Node* ShenandoahBarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& val.set_node(value); shenandoah_write_barrier_pre(kit, true /* do_load */, /*kit->control(),*/ access.base(), adr, adr_idx, val.node(), static_cast(val.type()), NULL /* pre_val */, access.type()); + + Node* result = BarrierSetC2::store_at_resolved(access, val); + bool is_array = (decorators & IS_ARRAY) != 0; + bool use_precise = is_array || anonymous; + post_barrier(kit, kit->control(), access.raw_access(), access.base(), adr, adr_idx, val.node(), access.type(), use_precise); + return result; } else { assert(access.is_opt_access(), "only for optimization passes"); assert(((decorators & C2_TIGHTLY_COUPLED_ALLOC) != 0 || !ShenandoahSATBBarrier) && (decorators & C2_ARRAY_COPY) != 0, "unexpected caller of this code"); @@ -528,8 +636,8 @@ Node* ShenandoahBarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& Node* enqueue = gvn.transform(new ShenandoahEnqueueBarrierNode(val.node())); val.set_node(enqueue); } + return BarrierSetC2::store_at_resolved(access, val); } - return BarrierSetC2::store_at_resolved(access, val); } Node* ShenandoahBarrierSetC2::load_at_resolved(C2Access& access, const Type* val_type) const { @@ -602,7 +710,7 @@ Node* ShenandoahBarrierSetC2::load_at_resolved(C2Access& access, const Type* val } Node* ShenandoahBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess& access, Node* expected_val, - Node* new_val, const Type* value_type) const { + Node* new_val, const Type* value_type) const { GraphKit* kit = access.kit(); if (access.is_oop()) { new_val = shenandoah_storeval_barrier(kit, new_val); @@ -644,6 +752,7 @@ Node* ShenandoahBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess } #endif load_store = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(NULL, load_store, false)); + post_barrier(kit, kit->control(), access.raw_access(), access.base(), access.addr().node(), access.alias_idx(), new_val, T_OBJECT, true); return load_store; } return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type); @@ -699,6 +808,8 @@ Node* ShenandoahBarrierSetC2::atomic_cmpxchg_bool_at_resolved(C2AtomicParseAcces } access.set_raw_access(load_store); pin_atomic_op(access); + post_barrier(kit, kit->control(), access.raw_access(), access.base(), + access.addr().node(), access.alias_idx(), new_val, T_OBJECT, true); return load_store; } return BarrierSetC2::atomic_cmpxchg_bool_at_resolved(access, expected_val, new_val, value_type); @@ -715,6 +826,8 @@ Node* ShenandoahBarrierSetC2::atomic_xchg_at_resolved(C2AtomicParseAccess& acces shenandoah_write_barrier_pre(kit, false /* do_load */, NULL, NULL, max_juint, NULL, NULL, result /* pre_val */, T_OBJECT); + post_barrier(kit, kit->control(), access.raw_access(), access.base(), + access.addr().node(), access.alias_idx(), val, T_OBJECT, true); } return result; } @@ -797,7 +910,7 @@ bool ShenandoahBarrierSetC2::clone_needs_barrier(Node* src, PhaseGVN& gvn) { } } else { return true; - } + } } else if (src_type->isa_aryptr()) { BasicType src_elem = src_type->klass()->as_array_klass()->element_type()->basic_type(); if (is_reference_type(src_elem)) { @@ -908,9 +1021,26 @@ void ShenandoahBarrierSetC2::unregister_potential_barrier_node(Node* node) const } } -void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* n) const { - if (is_shenandoah_wb_pre_call(n)) { - shenandoah_eliminate_wb_pre(n, ¯o->igvn()); +void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* node) const { + if (is_shenandoah_wb_pre_call(node)) { + shenandoah_eliminate_wb_pre(node, ¯o->igvn()); + } + if (node->Opcode() == Op_CastP2X && ShenandoahHeap::heap()->mode()->is_generational()) { + assert(node->Opcode() == Op_CastP2X, "ConvP2XNode required"); + Node *shift = node->unique_out(); + Node *addp = shift->unique_out(); + for (DUIterator_Last jmin, j = addp->last_outs(jmin); j >= jmin; --j) { + Node *mem = addp->last_out(j); + if (UseCondCardMark && mem->is_Load()) { + assert(mem->Opcode() == Op_LoadB, "unexpected code shape"); + // The load is checking if the card has been written so + // replace it with zero to fold the test. + macro->replace_node(mem, macro->intcon(0)); + continue; + } + assert(mem->is_Store(), "store required"); + macro->replace_node(mem, mem->in(MemNode::Memory)); + } } } diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp index db4940128058e..859fe59f5a4fc 100644 --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved. + * Copyright (c) 2018, 2020, Red Hat, Inc. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -73,6 +73,18 @@ class ShenandoahBarrierSetC2 : public BarrierSetC2 { Node* pre_val, BasicType bt) const; + Node* byte_map_base_node(GraphKit* kit) const; + + void post_barrier(GraphKit* kit, + Node* ctl, + Node* store, + Node* obj, + Node* adr, + uint adr_idx, + Node* val, + BasicType bt, + bool use_precise) const; + Node* shenandoah_enqueue_barrier(GraphKit* kit, Node* val) const; Node* shenandoah_storeval_barrier(GraphKit* kit, Node* obj) const; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp index dc9dd4ee39850..4da1a619a7361 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp @@ -52,7 +52,7 @@ static BarrierSetNMethod* make_barrier_set_nmethod(ShenandoahHeap* heap) { return new ShenandoahBarrierSetNMethod(heap); } -ShenandoahBarrierSet::ShenandoahBarrierSet(ShenandoahHeap* heap) : +ShenandoahBarrierSet::ShenandoahBarrierSet(ShenandoahHeap* heap, MemRegion heap_region) : BarrierSet(make_barrier_set_assembler(), make_barrier_set_c1(), make_barrier_set_c2(), @@ -62,6 +62,10 @@ ShenandoahBarrierSet::ShenandoahBarrierSet(ShenandoahHeap* heap) : _satb_mark_queue_buffer_allocator("SATB Buffer Allocator", ShenandoahSATBBufferSize), _satb_mark_queue_set(&_satb_mark_queue_buffer_allocator) { + if (heap->mode()->is_generational()) { + _card_table = new ShenandoahCardTable(heap_region); + _card_table->initialize(); + } } ShenandoahBarrierSetAssembler* ShenandoahBarrierSet::assembler() { @@ -185,3 +189,23 @@ void ShenandoahBarrierSet::clone_barrier_runtime(oop src) { } } +void ShenandoahBarrierSet::write_ref_array(HeapWord* start, size_t count) { + if (!_heap->mode()->is_generational()) { + return; + } + + HeapWord* end = (HeapWord*)((char*) start + (count * heapOopSize)); + // In the case of compressed oops, start and end may potentially be misaligned; + // so we need to conservatively align the first downward (this is not + // strictly necessary for current uses, but a case of good hygiene and, + // if you will, aesthetics) and the second upward (this is essential for + // current uses) to a HeapWord boundary, so we mark all cards overlapping + // this write. + HeapWord* aligned_start = align_down(start, HeapWordSize); + HeapWord* aligned_end = align_up (end, HeapWordSize); + // If compressed oops were not being used, these should already be aligned + assert(UseCompressedOops || (aligned_start == start && aligned_end == end), + "Expected heap word alignment of start and end"); + card_table()->dirty_MemRegion(MemRegion(aligned_start, aligned_end)); +} + diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp index 5d8194ae26d0a..9e339ae72dd89 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp @@ -27,6 +27,7 @@ #include "gc/shared/accessBarrierSupport.hpp" #include "gc/shared/barrierSet.hpp" +#include "gc/shenandoah/shenandoahCardTable.hpp" #include "gc/shenandoah/shenandoahHeap.hpp" #include "gc/shenandoah/shenandoahSATBMarkQueueSet.hpp" @@ -34,13 +35,13 @@ class ShenandoahBarrierSetAssembler; class ShenandoahBarrierSet: public BarrierSet { private: - ShenandoahHeap* _heap; + ShenandoahCardTable* _card_table; BufferNode::Allocator _satb_mark_queue_buffer_allocator; ShenandoahSATBMarkQueueSet _satb_mark_queue_set; public: - ShenandoahBarrierSet(ShenandoahHeap* heap); + ShenandoahBarrierSet(ShenandoahHeap* heap, MemRegion heap_region); static ShenandoahBarrierSetAssembler* assembler(); @@ -48,6 +49,8 @@ class ShenandoahBarrierSet: public BarrierSet { return barrier_set_cast(BarrierSet::barrier_set()); } + inline CardTable* card_table() { return _card_table; } + static ShenandoahSATBMarkQueueSet& satb_mark_queue_set() { return barrier_set()->_satb_mark_queue_set; } @@ -96,6 +99,11 @@ class ShenandoahBarrierSet: public BarrierSet { template inline oop load_reference_barrier_native(oop obj, T* load_addr); + template + void write_ref_field_post(T* field, oop newVal); + + void write_ref_array(HeapWord* start, size_t count); + private: template inline void arraycopy_marking(T* src, T* dst, size_t count); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp index b98b1168f3ade..62e668dad10d3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp @@ -26,6 +26,7 @@ #define SHARE_GC_SHENANDOAH_SHENANDOAHBARRIERSET_INLINE_HPP #include "gc/shared/barrierSet.hpp" +#include "gc/shared/cardTable.hpp" #include "gc/shenandoah/shenandoahAsserts.hpp" #include "gc/shenandoah/shenandoahBarrierSet.hpp" #include "gc/shenandoah/shenandoahCollectionSet.inline.hpp" @@ -35,6 +36,7 @@ #include "gc/shenandoah/shenandoahHeapRegion.hpp" #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp" #include "gc/shenandoah/shenandoahThreadLocalData.hpp" +#include "gc/shenandoah/mode/shenandoahMode.hpp" #include "memory/iterator.inline.hpp" #include "oops/oop.inline.hpp" @@ -160,6 +162,19 @@ inline void ShenandoahBarrierSet::keep_alive_if_weak(oop value) { } } +template +inline void ShenandoahBarrierSet::write_ref_field_post(T* field, oop newVal) { + if (ShenandoahHeap::heap()->mode()->is_generational()) { + volatile CardTable::CardValue* byte = card_table()->byte_for(field); + if (card_table()->scanned_concurrently()) { + // Perform a releasing store if the card table is scanned concurrently + Atomic::release_store(byte, CardTable::dirty_card_val()); + } else { + *byte = CardTable::dirty_card_val(); + } + } +} + template template inline oop ShenandoahBarrierSet::AccessBarrier::oop_load_not_in_heap(T* addr) { @@ -216,6 +231,7 @@ inline void ShenandoahBarrierSet::AccessBarrier::oop_st shenandoah_assert_not_in_cset_except (addr, value, value == NULL || ShenandoahHeap::heap()->cancelled_gc() || !ShenandoahHeap::heap()->is_concurrent_mark_in_progress()); oop_store_not_in_heap(addr, value); + ShenandoahBarrierSet::barrier_set()->write_ref_field_post(addr, value); } template @@ -249,7 +265,9 @@ inline oop ShenandoahBarrierSet::AccessBarrier::oop_ato template template inline oop ShenandoahBarrierSet::AccessBarrier::oop_atomic_cmpxchg_in_heap(T* addr, oop compare_value, oop new_value) { - return oop_atomic_cmpxchg_not_in_heap(addr, compare_value, new_value); + oop result = oop_atomic_cmpxchg_not_in_heap(addr, compare_value, new_value); + ShenandoahBarrierSet::barrier_set()->write_ref_field_post(addr, new_value); + return result; } template @@ -277,7 +295,9 @@ inline oop ShenandoahBarrierSet::AccessBarrier::oop_ato template template inline oop ShenandoahBarrierSet::AccessBarrier::oop_atomic_xchg_in_heap(T* addr, oop new_value) { - return oop_atomic_xchg_not_in_heap(addr, new_value); + oop result = oop_atomic_xchg_not_in_heap(addr, new_value); + ShenandoahBarrierSet::barrier_set()->write_ref_field_post(addr, new_value); + return result; } template @@ -299,11 +319,14 @@ template bool ShenandoahBarrierSet::AccessBarrier::oop_arraycopy_in_heap(arrayOop src_obj, size_t src_offset_in_bytes, T* src_raw, arrayOop dst_obj, size_t dst_offset_in_bytes, T* dst_raw, size_t length) { + T* src = arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw); + T* dst = arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw); + ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set(); - bs->arraycopy_barrier(arrayOopDesc::obj_offset_to_raw(src_obj, src_offset_in_bytes, src_raw), - arrayOopDesc::obj_offset_to_raw(dst_obj, dst_offset_in_bytes, dst_raw), - length); - return Raw::oop_arraycopy_in_heap(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length); + bs->arraycopy_barrier(src, dst, length); + bool result = Raw::oop_arraycopy_in_heap(src_obj, src_offset_in_bytes, src_raw, dst_obj, dst_offset_in_bytes, dst_raw, length); + bs->write_ref_array((HeapWord*) dst, length); + return result; } template diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCardTable.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCardTable.cpp new file mode 100644 index 0000000000000..b9fe2c3292e58 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahCardTable.cpp @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2020, Amazon.com, Inc. and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shenandoah/shenandoahCardTable.hpp" + +void ShenandoahCardTable::initialize() { + CardTable::initialize(); + resize_covered_region(_whole_heap); +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCardTable.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCardTable.hpp new file mode 100644 index 0000000000000..7a8c6c72c5039 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahCardTable.hpp @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2020, Amazon.com, Inc. and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHCARDTABLE_HPP +#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHCARDTABLE_HPP + +#include "gc/g1/g1RegionToSpaceMapper.hpp" +#include "gc/shared/cardTable.hpp" +#include "oops/oopsHierarchy.hpp" +#include "utilities/macros.hpp" + +class ShenandoahCardTable: public CardTable { + friend class VMStructs; + +public: + ShenandoahCardTable(MemRegion whole_heap): CardTable(whole_heap, /* conc_scan */ false) { } + + virtual void initialize(); + + inline bool is_in_young(oop obj) const { + ShouldNotReachHere(); + return false; + } +}; + +#endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHCARDTABLE_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 954661e442e71..1c59ea8fba645 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -34,6 +34,7 @@ #include "gc/shared/plab.hpp" #include "gc/shenandoah/shenandoahBarrierSet.hpp" +#include "gc/shenandoah/shenandoahCardTable.hpp" #include "gc/shenandoah/shenandoahClosures.inline.hpp" #include "gc/shenandoah/shenandoahCollectionSet.hpp" #include "gc/shenandoah/shenandoahCollectorPolicy.hpp" @@ -203,6 +204,28 @@ jint ShenandoahHeap::initialize() { "Cannot commit heap memory"); } + // + // After reserving the Java heap, create the card table, barriers, and workers, in dependency order + // + BarrierSet::set_barrier_set(new ShenandoahBarrierSet(this, _heap_region)); + + _workers = new ShenandoahWorkGang("Shenandoah GC Threads", _max_workers, + /* are_GC_task_threads */ true, + /* are_ConcurrentGC_threads */ true); + if (_workers == NULL) { + vm_exit_during_initialization("Failed necessary allocation."); + } else { + _workers->initialize_workers(); + } + + if (ParallelGCThreads > 1) { + _safepoint_workers = new ShenandoahWorkGang("Safepoint Cleanup Thread", + ParallelGCThreads, + /* are_GC_task_threads */ false, + /* are_ConcurrentGC_threads */ false); + _safepoint_workers->initialize_workers(); + } + // // Reserve and commit memory for bitmap(s) // @@ -455,7 +478,7 @@ ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) : _used(0), _committed(0), _bytes_allocated_since_gc_start(0), - _max_workers(MAX2(ConcGCThreads, ParallelGCThreads)), + _max_workers(MAX3(ConcGCThreads, ParallelGCThreads, 1U)), _workers(NULL), _safepoint_workers(NULL), _heap_region_special(false), @@ -488,25 +511,6 @@ ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) : _liveness_cache(NULL), _collection_set(NULL) { - BarrierSet::set_barrier_set(new ShenandoahBarrierSet(this)); - - _max_workers = MAX2(_max_workers, 1U); - _workers = new ShenandoahWorkGang("Shenandoah GC Threads", _max_workers, - /* are_GC_task_threads */ true, - /* are_ConcurrentGC_threads */ true); - if (_workers == NULL) { - vm_exit_during_initialization("Failed necessary allocation."); - } else { - _workers->initialize_workers(); - } - - if (ParallelGCThreads > 1) { - _safepoint_workers = new ShenandoahWorkGang("Safepoint Cleanup Thread", - ParallelGCThreads, - /* are_GC_task_threads */ false, - /* are_ConcurrentGC_threads */ false); - _safepoint_workers->initialize_workers(); - } } #ifdef _MSC_VER From 7e4a14c003efd87d0945bf09e4863b646fbb9e7e Mon Sep 17 00:00:00 2001 From: Bernd Mathiske Date: Mon, 16 Nov 2020 17:35:13 +0000 Subject: [PATCH 004/254] Generation labeling for regions and allocation requests. Reviewed-by: rkennke --- .../gc/shenandoah/shenandoahAllocRequest.hpp | 25 +++++++++++++------ .../share/gc/shenandoah/shenandoahFreeSet.cpp | 6 +++++ .../gc/shenandoah/shenandoahHeap.inline.hpp | 21 +++++++++++++--- .../gc/shenandoah/shenandoahHeapRegion.cpp | 17 ++++++++++++- .../gc/shenandoah/shenandoahHeapRegion.hpp | 10 ++++++++ 5 files changed, 68 insertions(+), 11 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahAllocRequest.hpp b/src/hotspot/share/gc/shenandoah/shenandoahAllocRequest.hpp index 59b4615d326c5..9b153231ec6ac 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahAllocRequest.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahAllocRequest.hpp @@ -27,6 +27,12 @@ #include "memory/allocation.hpp" +enum ShenandoahGeneration { + YOUNG_GEN, + OLD_GEN, + NO_GEN +}; + class ShenandoahAllocRequest : StackObj { public: enum Type { @@ -58,13 +64,14 @@ class ShenandoahAllocRequest : StackObj { size_t _requested_size; size_t _actual_size; Type _alloc_type; + ShenandoahGeneration const _generation; #ifdef ASSERT bool _actual_size_set; #endif - ShenandoahAllocRequest(size_t _min_size, size_t _requested_size, Type _alloc_type) : + ShenandoahAllocRequest(size_t _min_size, size_t _requested_size, Type _alloc_type, ShenandoahGeneration generation) : _min_size(_min_size), _requested_size(_requested_size), - _actual_size(0), _alloc_type(_alloc_type) + _actual_size(0), _alloc_type(_alloc_type), _generation(generation) #ifdef ASSERT , _actual_size_set(false) #endif @@ -72,19 +79,19 @@ class ShenandoahAllocRequest : StackObj { public: static inline ShenandoahAllocRequest for_tlab(size_t min_size, size_t requested_size) { - return ShenandoahAllocRequest(min_size, requested_size, _alloc_tlab); + return ShenandoahAllocRequest(min_size, requested_size, _alloc_tlab, YOUNG_GEN); } static inline ShenandoahAllocRequest for_gclab(size_t min_size, size_t requested_size) { - return ShenandoahAllocRequest(min_size, requested_size, _alloc_gclab); + return ShenandoahAllocRequest(min_size, requested_size, _alloc_gclab, YOUNG_GEN); } - static inline ShenandoahAllocRequest for_shared_gc(size_t requested_size) { - return ShenandoahAllocRequest(0, requested_size, _alloc_shared_gc); + static inline ShenandoahAllocRequest for_shared_gc(size_t requested_size, ShenandoahGeneration generation) { + return ShenandoahAllocRequest(0, requested_size, _alloc_shared_gc, generation); } static inline ShenandoahAllocRequest for_shared(size_t requested_size) { - return ShenandoahAllocRequest(0, requested_size, _alloc_shared); + return ShenandoahAllocRequest(0, requested_size, _alloc_shared, YOUNG_GEN); } inline size_t size() { @@ -158,6 +165,10 @@ class ShenandoahAllocRequest : StackObj { return false; } } + + ShenandoahGeneration generation() const { + return _generation; + } }; #endif // SHARE_GC_SHENANDOAH_SHENANDOAHALLOCREQUEST_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index bf63bf05a3ef8..52074c4aefc58 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -146,6 +146,12 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah return NULL; } + if (r->generation() == NO_GEN) { + r->set_generation(req.generation()); + } else if (r->generation() != req.generation()) { + return NULL; + } + try_recycle_trashed(r); in_new_region = r->is_empty(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp index 3bb97e01f593d..0436f3fadc744 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp @@ -260,12 +260,21 @@ inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) { assert(ShenandoahThreadLocalData::is_evac_allowed(thread), "must be enclosed in oom-evac scope"); - size_t size = p->size(); + ShenandoahGeneration target_gen = heap_region_containing(p)->generation(); + if (target_gen == YOUNG_GEN) { + if (ShenandoahForwarding::is_forwarded(p)) { + return ShenandoahBarrierSet::resolve_forwarded(p); + } else if (p->mark().age() > InitialTenuringThreshold) { + //tty->print_cr("promoting object: " PTR_FORMAT, p2i(p)); + //target_gen = OLD_GEN; + } + } assert(!heap_region_containing(p)->is_humongous(), "never evacuate humongous objects"); bool alloc_from_gclab = true; HeapWord* copy = NULL; + size_t size = p->size(); #ifdef ASSERT if (ShenandoahOOMDuringEvacALot && @@ -273,11 +282,11 @@ inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) { copy = NULL; } else { #endif - if (UseTLAB) { + if (UseTLAB && target_gen == YOUNG_GEN) { copy = allocate_from_gclab(thread, size); } if (copy == NULL) { - ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared_gc(size); + ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared_gc(size, target_gen); copy = allocate_memory(req); alloc_from_gclab = false; } @@ -302,6 +311,12 @@ inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) { if (result == copy_val) { // Successfully evacuated. Our copy is now the public one! shenandoah_assert_correct(NULL, copy_val); + + // Increment age in young copies + if (target_gen == YOUNG_GEN) { + copy_val->incr_age(); + } + return copy_val; } else { // Failed to evacuate. We need to deal with the object that is left behind. Since this diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index 2dd5f161d1df4..ea8927d99236b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -65,7 +65,8 @@ ShenandoahHeapRegion::ShenandoahHeapRegion(HeapWord* start, size_t index, bool c _gclab_allocs(0), _live_data(0), _critical_pins(0), - _update_watermark(start) { + _update_watermark(start), + _generation(NO_GEN) { assert(Universe::on_page_boundary(_bottom) && Universe::on_page_boundary(_end), "invalid space boundaries"); @@ -360,6 +361,19 @@ void ShenandoahHeapRegion::print_on(outputStream* st) const { default: ShouldNotReachHere(); } + switch (_generation) { + case YOUNG_GEN: + st->print("|Y"); + break; + case OLD_GEN: + st->print("|O"); + break; + case NO_GEN: + st->print("| "); + break; + default: + ShouldNotReachHere(); + } st->print("|BTE " INTPTR_FORMAT_W(12) ", " INTPTR_FORMAT_W(12) ", " INTPTR_FORMAT_W(12), p2i(bottom()), p2i(top()), p2i(end())); st->print("|TAMS " INTPTR_FORMAT_W(12), @@ -429,6 +443,7 @@ void ShenandoahHeapRegion::recycle() { set_update_watermark(bottom()); make_empty(); + _generation = NO_GEN; if (ZapUnusedHeapArea) { SpaceMangler::mangle_region(MemRegion(bottom(), end())); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp index 766ab496c446e..9ee07e3f15b90 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp @@ -245,6 +245,8 @@ class ShenandoahHeapRegion { HeapWord* volatile _update_watermark; + ShenandoahGeneration _generation; + public: ShenandoahHeapRegion(HeapWord* start, size_t index, bool committed); @@ -386,6 +388,14 @@ class ShenandoahHeapRegion { inline void set_update_watermark(HeapWord* w); inline void set_update_watermark_at_safepoint(HeapWord* w); + ShenandoahGeneration generation() const { + return _generation; + } + + void set_generation(ShenandoahGeneration generation) { + _generation = generation; + } + private: void do_commit(); void do_uncommit(); From 6620f0eb6c728db2bc9ae9161ff606127b073b68 Mon Sep 17 00:00:00 2001 From: Bernd Mathiske Date: Wed, 18 Nov 2020 10:03:10 +0000 Subject: [PATCH 005/254] Generation affiliation transitions for heap regions. Reviewed-by: rkennke --- .../gc/shenandoah/shenandoahAllocRequest.hpp | 30 +++---- .../share/gc/shenandoah/shenandoahFreeSet.cpp | 8 +- .../gc/shenandoah/shenandoahGeneration.cpp | 26 ++++++ .../gc/shenandoah/shenandoahGeneration.hpp | 39 +++++++++ .../share/gc/shenandoah/shenandoahHeap.cpp | 4 + .../share/gc/shenandoah/shenandoahHeap.hpp | 2 + .../gc/shenandoah/shenandoahHeap.inline.hpp | 8 +- .../gc/shenandoah/shenandoahHeapRegion.cpp | 82 +++++++++++++++++-- .../gc/shenandoah/shenandoahHeapRegion.hpp | 12 ++- 9 files changed, 172 insertions(+), 39 deletions(-) create mode 100644 src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp create mode 100644 src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp diff --git a/src/hotspot/share/gc/shenandoah/shenandoahAllocRequest.hpp b/src/hotspot/share/gc/shenandoah/shenandoahAllocRequest.hpp index 9b153231ec6ac..4f3ec22b6ef47 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahAllocRequest.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahAllocRequest.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved. + * Copyright (c) 2018, 2020, Red Hat, Inc. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -27,10 +27,10 @@ #include "memory/allocation.hpp" -enum ShenandoahGeneration { - YOUNG_GEN, - OLD_GEN, - NO_GEN +enum ShenandoahRegionAffiliation { + FREE, + YOUNG_GENERATION, + OLD_GENERATION }; class ShenandoahAllocRequest : StackObj { @@ -64,14 +64,14 @@ class ShenandoahAllocRequest : StackObj { size_t _requested_size; size_t _actual_size; Type _alloc_type; - ShenandoahGeneration const _generation; + ShenandoahRegionAffiliation const _affiliation; #ifdef ASSERT bool _actual_size_set; #endif - ShenandoahAllocRequest(size_t _min_size, size_t _requested_size, Type _alloc_type, ShenandoahGeneration generation) : + ShenandoahAllocRequest(size_t _min_size, size_t _requested_size, Type _alloc_type, ShenandoahRegionAffiliation affiliation) : _min_size(_min_size), _requested_size(_requested_size), - _actual_size(0), _alloc_type(_alloc_type), _generation(generation) + _actual_size(0), _alloc_type(_alloc_type), _affiliation(affiliation) #ifdef ASSERT , _actual_size_set(false) #endif @@ -79,19 +79,19 @@ class ShenandoahAllocRequest : StackObj { public: static inline ShenandoahAllocRequest for_tlab(size_t min_size, size_t requested_size) { - return ShenandoahAllocRequest(min_size, requested_size, _alloc_tlab, YOUNG_GEN); + return ShenandoahAllocRequest(min_size, requested_size, _alloc_tlab, YOUNG_GENERATION); } static inline ShenandoahAllocRequest for_gclab(size_t min_size, size_t requested_size) { - return ShenandoahAllocRequest(min_size, requested_size, _alloc_gclab, YOUNG_GEN); + return ShenandoahAllocRequest(min_size, requested_size, _alloc_gclab, YOUNG_GENERATION); } - static inline ShenandoahAllocRequest for_shared_gc(size_t requested_size, ShenandoahGeneration generation) { - return ShenandoahAllocRequest(0, requested_size, _alloc_shared_gc, generation); + static inline ShenandoahAllocRequest for_shared_gc(size_t requested_size, ShenandoahRegionAffiliation affiliation) { + return ShenandoahAllocRequest(0, requested_size, _alloc_shared_gc, affiliation); } static inline ShenandoahAllocRequest for_shared(size_t requested_size) { - return ShenandoahAllocRequest(0, requested_size, _alloc_shared, YOUNG_GEN); + return ShenandoahAllocRequest(0, requested_size, _alloc_shared, YOUNG_GENERATION); } inline size_t size() { @@ -166,8 +166,8 @@ class ShenandoahAllocRequest : StackObj { } } - ShenandoahGeneration generation() const { - return _generation; + ShenandoahRegionAffiliation affiliation() const { + return _affiliation; } }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index 52074c4aefc58..4bafad82939b5 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, 2019, Red Hat, Inc. All rights reserved. + * Copyright (c) 2016, 2020, Red Hat, Inc. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -146,9 +146,9 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah return NULL; } - if (r->generation() == NO_GEN) { - r->set_generation(req.generation()); - } else if (r->generation() != req.generation()) { + if (r->affiliation() == FREE) { + r->set_affiliation(req.affiliation()); + } else if (r->affiliation() != req.affiliation()) { return NULL; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp new file mode 100644 index 0000000000000..24c87caa50182 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2020, Amazon.com, Inc. and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shenandoah/shenandoahGeneration.hpp" diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp new file mode 100644 index 0000000000000..e016bf1e9c922 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2020, Amazon.com, Inc. and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHGENERATION_HPP +#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHGENERATION_HPP + +class ShenandoahGeneration { +public: + ShenandoahGeneration(); +}; + +class ShenandoahYoungGeneration : public ShenandoahGeneration { +}; + +class ShenandoahOldGeneration : public ShenandoahGeneration { +}; + +#endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHGENERATION_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 1c59ea8fba645..140ab89609724 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -716,6 +716,10 @@ bool ShenandoahHeap::is_in(const void* p) const { return p >= heap_base && p < last_region_end; } +bool ShenandoahHeap::is_in_young(const void* p) const { + return heap_region_containing(p)->affiliation() == YOUNG_GENERATION; +} + void ShenandoahHeap::op_uncommit(double shrink_before, size_t shrink_until) { assert (ShenandoahUncommit, "should be enabled"); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index 494166a640e5a..9af723e8afb0b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -547,6 +547,8 @@ class ShenandoahHeap : public CollectedHeap { bool is_in(const void* p) const; + bool is_in_young(const void* p) const; + MemRegion reserved_region() const { return _reserved; } bool is_in_reserved(const void* addr) const { return _reserved.contains(addr); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp index 0436f3fadc744..634c0b1dc9489 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp @@ -260,8 +260,8 @@ inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) { assert(ShenandoahThreadLocalData::is_evac_allowed(thread), "must be enclosed in oom-evac scope"); - ShenandoahGeneration target_gen = heap_region_containing(p)->generation(); - if (target_gen == YOUNG_GEN) { + ShenandoahRegionAffiliation target_gen = heap_region_containing(p)->affiliation(); + if (target_gen == YOUNG_GENERATION) { if (ShenandoahForwarding::is_forwarded(p)) { return ShenandoahBarrierSet::resolve_forwarded(p); } else if (p->mark().age() > InitialTenuringThreshold) { @@ -282,7 +282,7 @@ inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) { copy = NULL; } else { #endif - if (UseTLAB && target_gen == YOUNG_GEN) { + if (UseTLAB && target_gen == YOUNG_GENERATION) { copy = allocate_from_gclab(thread, size); } if (copy == NULL) { @@ -313,7 +313,7 @@ inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) { shenandoah_assert_correct(NULL, copy_val); // Increment age in young copies - if (target_gen == YOUNG_GEN) { + if (target_gen == YOUNG_GENERATION) { copy_val->incr_age(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index ea8927d99236b..ef2344b937b04 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2019, Red Hat, Inc. All rights reserved. + * Copyright (c) 2013, 2020, Red Hat, Inc. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -24,6 +24,7 @@ #include "precompiled.hpp" #include "memory/allocation.hpp" +#include "gc/shenandoah/shenandoahCardTable.hpp" #include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahHeapRegion.hpp" @@ -66,7 +67,7 @@ ShenandoahHeapRegion::ShenandoahHeapRegion(HeapWord* start, size_t index, bool c _live_data(0), _critical_pins(0), _update_watermark(start), - _generation(NO_GEN) { + _affiliation(ShenandoahRegionAffiliation::FREE) { assert(Universe::on_page_boundary(_bottom) && Universe::on_page_boundary(_end), "invalid space boundaries"); @@ -361,16 +362,16 @@ void ShenandoahHeapRegion::print_on(outputStream* st) const { default: ShouldNotReachHere(); } - switch (_generation) { - case YOUNG_GEN: + switch (_affiliation) { + case FREE: + st->print("|F"); + break; + case YOUNG_GENERATION: st->print("|Y"); break; - case OLD_GEN: + case OLD_GENERATION: st->print("|O"); break; - case NO_GEN: - st->print("| "); - break; default: ShouldNotReachHere(); } @@ -443,7 +444,7 @@ void ShenandoahHeapRegion::recycle() { set_update_watermark(bottom()); make_empty(); - _generation = NO_GEN; + _affiliation = ShenandoahRegionAffiliation::FREE; if (ZapUnusedHeapArea) { SpaceMangler::mangle_region(MemRegion(bottom(), end())); @@ -677,3 +678,66 @@ void ShenandoahHeapRegion::record_unpin() { size_t ShenandoahHeapRegion::pin_count() const { return Atomic::load(&_critical_pins); } + +class UpdateOopCardValueClosure : public BasicOopIterateClosure { + CardTable* _card_table; + +public: + UpdateOopCardValueClosure(CardTable *card_table) : _card_table(card_table) { } + + void do_oop(oop* p) { + if (ShenandoahHeap::heap()->is_in_young(*p)) { + volatile CardTable::CardValue* card_value = _card_table->byte_for(*p); + *card_value = CardTable::dirty_card_val(); + } + } + + void do_oop(narrowOop* p) { + ShouldNotReachHere(); + } +}; + +class UpdateObjectCardValuesClosure : public BasicOopIterateClosure { + UpdateOopCardValueClosure* _oop_closure; + +public: + UpdateObjectCardValuesClosure(UpdateOopCardValueClosure *oop_closure) : + _oop_closure(oop_closure) { + } + + void do_oop(oop* p) { + (*p)->oop_iterate(_oop_closure); + } + + void do_oop(narrowOop* p) { + ShouldNotReachHere(); + } +}; + +void ShenandoahHeapRegion::set_affiliation(ShenandoahRegionAffiliation new_affiliation) { + if (_affiliation == new_affiliation) { + return; + } + CardTable* card_table = ShenandoahBarrierSet::barrier_set()->card_table(); + switch (new_affiliation) { + case FREE: + card_table->clear_MemRegion(MemRegion(_bottom, _end)); + break; + case YOUNG_GENERATION: + break; + case OLD_GENERATION: + if (_affiliation == YOUNG_GENERATION) { + assert(SafepointSynchronize::is_at_safepoint(), "old gen card values must be updated in a safepoint"); + card_table->clear_MemRegion(MemRegion(_bottom, _end)); + + UpdateOopCardValueClosure oop_closure(card_table); + UpdateObjectCardValuesClosure object_closure(&oop_closure); + oop_iterate(&object_closure); + } + break; + default: + ShouldNotReachHere(); + return; + } + _affiliation = new_affiliation; +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp index 9ee07e3f15b90..59caa8122700e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2019, Red Hat, Inc. All rights reserved. + * Copyright (c) 2013, 2020, Red Hat, Inc. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -245,7 +245,7 @@ class ShenandoahHeapRegion { HeapWord* volatile _update_watermark; - ShenandoahGeneration _generation; + ShenandoahRegionAffiliation _affiliation; public: ShenandoahHeapRegion(HeapWord* start, size_t index, bool committed); @@ -388,13 +388,11 @@ class ShenandoahHeapRegion { inline void set_update_watermark(HeapWord* w); inline void set_update_watermark_at_safepoint(HeapWord* w); - ShenandoahGeneration generation() const { - return _generation; + ShenandoahRegionAffiliation affiliation() const { + return _affiliation; } - void set_generation(ShenandoahGeneration generation) { - _generation = generation; - } + void set_affiliation(ShenandoahRegionAffiliation new_affiliation); private: void do_commit(); From fe1fe0b190edf258713e0b01aafba1d61d930573 Mon Sep 17 00:00:00 2001 From: Bernd Mathiske Date: Wed, 9 Dec 2020 15:43:34 +0000 Subject: [PATCH 006/254] Differentiate young generation marking. Reviewed-by: rkennke --- .../shenandoah/shenandoahConcurrentMark.cpp | 173 ++++++++++++------ .../shenandoah/shenandoahConcurrentMark.hpp | 39 +++- .../shenandoahConcurrentMark.inline.hpp | 5 +- .../gc/shenandoah/shenandoahControlThread.cpp | 18 +- .../gc/shenandoah/shenandoahControlThread.hpp | 2 +- .../share/gc/shenandoah/shenandoahFreeSet.cpp | 4 +- .../gc/shenandoah/shenandoahGeneration.hpp | 19 +- .../shenandoah/shenandoahGlobalGeneration.hpp | 37 ++++ .../share/gc/shenandoah/shenandoahHeap.cpp | 48 ++--- .../share/gc/shenandoah/shenandoahHeap.hpp | 24 +-- .../gc/shenandoah/shenandoahMarkCompact.cpp | 7 +- .../gc/shenandoah/shenandoahOopClosures.hpp | 34 +++- .../shenandoahOopClosures.inline.hpp | 5 +- .../gc/shenandoah/shenandoahVMOperations.cpp | 5 +- .../gc/shenandoah/shenandoahVMOperations.hpp | 8 +- ...tion.cpp => shenandoahYoungGeneration.hpp} | 11 +- 16 files changed, 309 insertions(+), 130 deletions(-) create mode 100644 src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp rename src/hotspot/share/gc/shenandoah/{shenandoahGeneration.cpp => shenandoahYoungGeneration.hpp} (77%) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp index 5331ca26d4859..f4ad0dcfaeb92 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp @@ -51,7 +51,7 @@ #include "oops/oop.inline.hpp" #include "runtime/handles.inline.hpp" -template +template class ShenandoahInitMarkRootsClosure : public OopClosure { private: ShenandoahObjToScanQueue* _queue; @@ -60,7 +60,7 @@ class ShenandoahInitMarkRootsClosure : public OopClosure { template inline void do_oop_work(T* p) { - ShenandoahConcurrentMark::mark_through_ref(p, _heap, _queue, _mark_context, false); + ShenandoahConcurrentMark::mark_through_ref(p, _heap, _queue, _mark_context, false); } public: @@ -84,10 +84,12 @@ ShenandoahMarkRefsSuperClosure::ShenandoahMarkRefsSuperClosure(ShenandoahObjToSc template class ShenandoahInitMarkRootsTask : public AbstractGangTask { private: - ShenandoahRootScanner* _rp; + ShenandoahConcurrentMark* const _scm; + ShenandoahRootScanner* const _rp; public: - ShenandoahInitMarkRootsTask(ShenandoahRootScanner* rp) : + ShenandoahInitMarkRootsTask(ShenandoahConcurrentMark* scm, ShenandoahRootScanner* rp) : AbstractGangTask("Shenandoah Init Mark Roots"), + _scm(scm), _rp(rp) { } @@ -96,13 +98,27 @@ class ShenandoahInitMarkRootsTask : public AbstractGangTask { ShenandoahParallelWorkerSession worker_session(worker_id); ShenandoahHeap* heap = ShenandoahHeap::heap(); - ShenandoahObjToScanQueueSet* queues = heap->concurrent_mark()->task_queues(); + ShenandoahObjToScanQueueSet* queues = _scm->task_queues(); assert(queues->get_reserved() > worker_id, "Queue has not been reserved for worker id: %d", worker_id); ShenandoahObjToScanQueue* q = queues->queue(worker_id); - ShenandoahInitMarkRootsClosure mark_cl(q); - do_work(heap, &mark_cl, worker_id); + switch (_scm->generation_mode()) { + case YOUNG: { + ShenandoahInitMarkRootsClosure mark_cl(q); + do_work(heap, &mark_cl, worker_id); + break; + } + case GLOBAL: { + ShenandoahInitMarkRootsClosure mark_cl(q); + do_work(heap, &mark_cl, worker_id); + break; + } + default: { + ShouldNotReachHere(); + break; + } + } } private: @@ -161,15 +177,16 @@ class ShenandoahConcurrentMarkingTask : public AbstractGangTask { } }; +template class ShenandoahSATBAndRemarkCodeRootsThreadsClosure : public ThreadClosure { private: - ShenandoahSATBBufferClosure* _satb_cl; + ShenandoahSATBBufferClosure* _satb_cl; OopClosure* const _cl; MarkingCodeBlobClosure* _code_cl; uintx _claim_token; public: - ShenandoahSATBAndRemarkCodeRootsThreadsClosure(ShenandoahSATBBufferClosure* satb_cl, OopClosure* cl, MarkingCodeBlobClosure* code_cl) : + ShenandoahSATBAndRemarkCodeRootsThreadsClosure(ShenandoahSATBBufferClosure* satb_cl, OopClosure* cl, MarkingCodeBlobClosure* code_cl) : _satb_cl(satb_cl), _cl(cl), _code_cl(code_cl), _claim_token(Threads::thread_claim_token()) {} @@ -227,6 +244,7 @@ void ShenandoahProcessConcurrentRootsTask::work(uint worker_id) { _rs.oops_do(&cl, worker_id); } +template class ShenandoahFinalMarkingTask : public AbstractGangTask { private: ShenandoahConcurrentMark* _cm; @@ -252,23 +270,23 @@ class ShenandoahFinalMarkingTask : public AbstractGangTask { { ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id); - ShenandoahSATBBufferClosure cl(q); + ShenandoahSATBBufferClosure cl(q); SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set(); while (satb_mq_set.apply_closure_to_completed_buffer(&cl)); bool do_nmethods = heap->unload_classes() && !ShenandoahConcurrentRoots::can_do_concurrent_class_unloading(); if (heap->has_forwarded_objects()) { - ShenandoahMarkResolveRefsClosure resolve_mark_cl(q, rp); + ShenandoahMarkResolveRefsClosure resolve_mark_cl(q, rp); MarkingCodeBlobClosure blobsCl(&resolve_mark_cl, !CodeBlobToOopClosure::FixRelocations); - ShenandoahSATBAndRemarkCodeRootsThreadsClosure tc(&cl, - ShenandoahStoreValEnqueueBarrier ? &resolve_mark_cl : NULL, - do_nmethods ? &blobsCl : NULL); + ShenandoahSATBAndRemarkCodeRootsThreadsClosure tc(&cl, + ShenandoahStoreValEnqueueBarrier ? &resolve_mark_cl : NULL, + do_nmethods ? &blobsCl : NULL); Threads::threads_do(&tc); } else { - ShenandoahMarkRefsClosure mark_cl(q, rp); + ShenandoahMarkRefsClosure mark_cl(q, rp); MarkingCodeBlobClosure blobsCl(&mark_cl, !CodeBlobToOopClosure::FixRelocations); - ShenandoahSATBAndRemarkCodeRootsThreadsClosure tc(&cl, - ShenandoahStoreValEnqueueBarrier ? &mark_cl : NULL, - do_nmethods ? &blobsCl : NULL); + ShenandoahSATBAndRemarkCodeRootsThreadsClosure tc(&cl, + ShenandoahStoreValEnqueueBarrier ? &mark_cl : NULL, + do_nmethods ? &blobsCl : NULL); Threads::threads_do(&tc); } } @@ -302,12 +320,12 @@ void ShenandoahConcurrentMark::mark_roots(ShenandoahPhaseTimings::Phase root_pha task_queues()->reserve(nworkers); if (heap->has_forwarded_objects()) { - ShenandoahInitMarkRootsTask mark_roots(&root_proc); + ShenandoahInitMarkRootsTask mark_roots(this, &root_proc); workers->run_task(&mark_roots); } else { // No need to update references, which means the heap is stable. // Can save time not walking through forwarding pointers. - ShenandoahInitMarkRootsTask mark_roots(&root_proc); + ShenandoahInitMarkRootsTask mark_roots(this, &root_proc); workers->run_task(&mark_roots); } } @@ -393,24 +411,28 @@ void ShenandoahConcurrentMark::initialize(uint workers) { // Mark concurrent roots during concurrent phases class ShenandoahMarkConcurrentRootsTask : public AbstractGangTask { private: + ShenandoahConcurrentMark* _scm; SuspendibleThreadSetJoiner _sts_joiner; ShenandoahConcurrentRootScanner _rs; ShenandoahObjToScanQueueSet* const _queue_set; ShenandoahReferenceProcessor* const _rp; public: - ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs, + ShenandoahMarkConcurrentRootsTask(ShenandoahConcurrentMark* scm, + ShenandoahObjToScanQueueSet* qs, ShenandoahReferenceProcessor* rp, ShenandoahPhaseTimings::Phase phase, uint nworkers); void work(uint worker_id); }; -ShenandoahMarkConcurrentRootsTask::ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs, +ShenandoahMarkConcurrentRootsTask::ShenandoahMarkConcurrentRootsTask(ShenandoahConcurrentMark* scm, + ShenandoahObjToScanQueueSet* qs, ShenandoahReferenceProcessor* rp, ShenandoahPhaseTimings::Phase phase, uint nworkers) : AbstractGangTask("Shenandoah Concurrent Mark Roots"), + _scm(scm), _rs(nworkers, phase), _queue_set(qs), _rp(rp) { @@ -420,8 +442,22 @@ ShenandoahMarkConcurrentRootsTask::ShenandoahMarkConcurrentRootsTask(ShenandoahO void ShenandoahMarkConcurrentRootsTask::work(uint worker_id) { ShenandoahConcurrentWorkerSession worker_session(worker_id); ShenandoahObjToScanQueue* q = _queue_set->queue(worker_id); - ShenandoahMarkResolveRefsClosure cl(q, _rp); - _rs.oops_do(&cl, worker_id); + switch (_scm->generation_mode()) { + case YOUNG: { + ShenandoahMarkResolveRefsClosure cl(q, _rp); + _rs.oops_do(&cl, worker_id); + break; + } + case GLOBAL: { + ShenandoahMarkResolveRefsClosure cl(q, _rp); + _rs.oops_do(&cl, worker_id); + break; + } + default: { + ShouldNotReachHere(); + break; + } + } } void ShenandoahConcurrentMark::mark_from_roots() { @@ -435,7 +471,7 @@ void ShenandoahConcurrentMark::mark_from_roots() { { ShenandoahGCPhase phase(ShenandoahPhaseTimings::conc_mark_roots); // Use separate task to mark concurrent roots, since it may hold ClassLoaderData_lock and CodeCache_lock - ShenandoahMarkConcurrentRootsTask task(task_queues(), rp, ShenandoahPhaseTimings::conc_mark_roots, nworkers); + ShenandoahMarkConcurrentRootsTask task(this, task_queues(), rp, ShenandoahPhaseTimings::conc_mark_roots, nworkers); workers->run_task(&task); } @@ -463,12 +499,31 @@ void ShenandoahConcurrentMark::finish_mark_from_roots(bool full_gc) { ShenandoahPhaseTimings::full_gc_scan_conc_roots : ShenandoahPhaseTimings::degen_gc_scan_conc_roots; ShenandoahGCPhase gc_phase(phase); - if (_heap->has_forwarded_objects()) { - ShenandoahProcessConcurrentRootsTask task(this, phase, nworkers); - _heap->workers()->run_task(&task); - } else { - ShenandoahProcessConcurrentRootsTask task(this, phase, nworkers); - _heap->workers()->run_task(&task); + switch (generation_mode()) { + case YOUNG: { + if (_heap->has_forwarded_objects()) { + ShenandoahProcessConcurrentRootsTask> task(this, phase, nworkers); + _heap->workers()->run_task(&task); + } else { + ShenandoahProcessConcurrentRootsTask> task(this, phase, nworkers); + _heap->workers()->run_task(&task); + } + break; + } + case GLOBAL: { + if (_heap->has_forwarded_objects()) { + ShenandoahProcessConcurrentRootsTask> task(this, phase, nworkers); + _heap->workers()->run_task(&task); + } else { + ShenandoahProcessConcurrentRootsTask> task(this, phase, nworkers); + _heap->workers()->run_task(&task); + } + break; + } + default: { + ShouldNotReachHere(); + break; + } } } @@ -487,8 +542,22 @@ void ShenandoahConcurrentMark::finish_mark_from_roots(bool full_gc) { StrongRootsScope scope(nworkers); TaskTerminator terminator(nworkers, task_queues()); - ShenandoahFinalMarkingTask task(this, &terminator, ShenandoahStringDedup::is_enabled()); - _heap->workers()->run_task(&task); + switch (generation_mode()) { + case YOUNG: { + ShenandoahFinalMarkingTask task(this, &terminator, ShenandoahStringDedup::is_enabled()); + _heap->workers()->run_task(&task); + break; + } + case GLOBAL: { + ShenandoahFinalMarkingTask task(this, &terminator, ShenandoahStringDedup::is_enabled()); + _heap->workers()->run_task(&task); + break; + } + default: { + ShouldNotReachHere(); + break; + } + } } assert(task_queues()->is_empty(), "Should be empty"); @@ -513,7 +582,7 @@ ShenandoahObjToScanQueue* ShenandoahConcurrentMark::get_queue(uint worker_id) { return _task_queues->queue(worker_id); } -template +template void ShenandoahConcurrentMark::mark_loop_prework(uint w, TaskTerminator *t, ShenandoahReferenceProcessor* rp, bool strdedup) { ShenandoahObjToScanQueue* q = get_queue(w); @@ -525,37 +594,37 @@ void ShenandoahConcurrentMark::mark_loop_prework(uint w, TaskTerminator *t, Shen if (_heap->unload_classes()) { if (_heap->has_forwarded_objects()) { if (strdedup) { - ShenandoahMarkUpdateRefsMetadataDedupClosure cl(q, rp); - mark_loop_work(&cl, ld, w, t); + ShenandoahMarkUpdateRefsMetadataDedupClosure cl(q, rp); + mark_loop_work, GENERATION, CANCELLABLE>(&cl, ld, w, t); } else { - ShenandoahMarkUpdateRefsMetadataClosure cl(q, rp); - mark_loop_work(&cl, ld, w, t); + ShenandoahMarkUpdateRefsMetadataClosure cl(q, rp); + mark_loop_work, GENERATION, CANCELLABLE>(&cl, ld, w, t); } } else { if (strdedup) { - ShenandoahMarkRefsMetadataDedupClosure cl(q, rp); - mark_loop_work(&cl, ld, w, t); + ShenandoahMarkRefsMetadataDedupClosure cl(q, rp); + mark_loop_work, GENERATION, CANCELLABLE>(&cl, ld, w, t); } else { - ShenandoahMarkRefsMetadataClosure cl(q, rp); - mark_loop_work(&cl, ld, w, t); + ShenandoahMarkRefsMetadataClosure cl(q, rp); + mark_loop_work, GENERATION, CANCELLABLE>(&cl, ld, w, t); } } } else { if (_heap->has_forwarded_objects()) { if (strdedup) { - ShenandoahMarkUpdateRefsDedupClosure cl(q, rp); - mark_loop_work(&cl, ld, w, t); + ShenandoahMarkUpdateRefsDedupClosure cl(q, rp); + mark_loop_work, GENERATION, CANCELLABLE>(&cl, ld, w, t); } else { - ShenandoahMarkUpdateRefsClosure cl(q, rp); - mark_loop_work(&cl, ld, w, t); + ShenandoahMarkUpdateRefsClosure cl(q, rp); + mark_loop_work, GENERATION, CANCELLABLE>(&cl, ld, w, t); } } else { if (strdedup) { - ShenandoahMarkRefsDedupClosure cl(q, rp); - mark_loop_work(&cl, ld, w, t); + ShenandoahMarkRefsDedupClosure cl(q, rp); + mark_loop_work, GENERATION, CANCELLABLE>(&cl, ld, w, t); } else { - ShenandoahMarkRefsClosure cl(q, rp); - mark_loop_work(&cl, ld, w, t); + ShenandoahMarkRefsClosure cl(q, rp); + mark_loop_work, GENERATION, CANCELLABLE>(&cl, ld, w, t); } } } @@ -563,7 +632,7 @@ void ShenandoahConcurrentMark::mark_loop_prework(uint w, TaskTerminator *t, Shen _heap->flush_liveness_cache(w); } -template +template void ShenandoahConcurrentMark::mark_loop_work(T* cl, ShenandoahLiveData* live_data, uint worker_id, TaskTerminator *terminator) { uintx stride = ShenandoahMarkLoopStride; @@ -602,7 +671,7 @@ void ShenandoahConcurrentMark::mark_loop_work(T* cl, ShenandoahLiveData* live_da } q = get_queue(worker_id); - ShenandoahSATBBufferClosure drain_satb(q); + ShenandoahSATBBufferClosure drain_satb(q); SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set(); /* diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp index 2e8538234eea8..fa9dd708291ee 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp @@ -36,13 +36,18 @@ class ShenandoahReferenceProcessor; class ShenandoahConcurrentMark: public CHeapObj { private: + GenerationMode const _generation_mode; ShenandoahHeap* _heap; ShenandoahObjToScanQueueSet* _task_queues; public: + ShenandoahConcurrentMark(GenerationMode generation_mode) : _generation_mode(generation_mode) { } + void initialize(uint workers); void cancel(); + GenerationMode generation_mode() const { return _generation_mode; } + // ---------- Marking loop and tasks // private: @@ -57,23 +62,39 @@ class ShenandoahConcurrentMark: public CHeapObj { inline void count_liveness(ShenandoahLiveData* live_data, oop obj); - template + template void mark_loop_work(T* cl, ShenandoahLiveData* live_data, uint worker_id, TaskTerminator *t); - template + template void mark_loop_prework(uint worker_id, TaskTerminator *terminator, ShenandoahReferenceProcessor* rp, bool strdedup); public: - void mark_loop(uint worker_id, TaskTerminator* terminator, ShenandoahReferenceProcessor* rp, - bool cancellable, bool strdedup) { - if (cancellable) { - mark_loop_prework(worker_id, terminator, rp, strdedup); - } else { - mark_loop_prework(worker_id, terminator, rp, strdedup); + void mark_loop(uint worker_id, TaskTerminator* terminator, ShenandoahReferenceProcessor *rp, bool cancellable, bool strdedup) { + switch (generation_mode()) { + case YOUNG: { + if (cancellable) { + mark_loop_prework(worker_id, terminator, rp, strdedup); + } else { + mark_loop_prework(worker_id, terminator, rp, strdedup); + } + break; + } + case GLOBAL: { + if (cancellable) { + mark_loop_prework(worker_id, terminator, rp, strdedup); + } else { + mark_loop_prework(worker_id, terminator, rp, strdedup); + } + break; + } + default: { + ShouldNotReachHere(); + break; + } } } - template + template static inline void mark_through_ref(T* p, ShenandoahHeap* heap, ShenandoahObjToScanQueue* q, ShenandoahMarkingContext* const mark_context, bool weak); void mark_from_roots(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.inline.hpp index 40e0957fa72e1..ecf3bfc8f4cfc 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.inline.hpp @@ -202,6 +202,7 @@ inline void ShenandoahConcurrentMark::do_chunked_array(ShenandoahObjToScanQueue* array->oop_iterate_range(cl, from, to); } +template class ShenandoahSATBBufferClosure : public SATBBufferClosure { private: ShenandoahObjToScanQueue* _queue; @@ -228,12 +229,12 @@ class ShenandoahSATBBufferClosure : public SATBBufferClosure { void do_buffer_impl(void **buffer, size_t size) { for (size_t i = 0; i < size; ++i) { oop *p = (oop *) &buffer[i]; - ShenandoahConcurrentMark::mark_through_ref(p, _heap, _queue, _mark_context, false); + ShenandoahConcurrentMark::mark_through_ref(p, _heap, _queue, _mark_context, false); } } }; -template +template inline void ShenandoahConcurrentMark::mark_through_ref(T *p, ShenandoahHeap* heap, ShenandoahObjToScanQueue* q, ShenandoahMarkingContext* const mark_context, bool weak) { T o = RawAccess<>::oop_load(p); if (!CompressedOops::is_null(o)) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index 706fee202ef2d..7c4f9d52c8404 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -28,6 +28,7 @@ #include "gc/shenandoah/shenandoahCollectorPolicy.hpp" #include "gc/shenandoah/shenandoahControlThread.hpp" #include "gc/shenandoah/shenandoahFreeSet.hpp" +#include "gc/shenandoah/shenandoahGeneration.hpp" #include "gc/shenandoah/shenandoahPhaseTimings.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahMonitoringSupport.hpp" @@ -36,6 +37,7 @@ #include "gc/shenandoah/shenandoahVMOperations.hpp" #include "gc/shenandoah/shenandoahWorkerPolicy.hpp" #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" +#include "gc/shenandoah/mode/shenandoahMode.hpp" #include "memory/iterator.hpp" #include "memory/universe.hpp" #include "runtime/atomic.hpp" @@ -203,7 +205,13 @@ void ShenandoahControlThread::run_service() { switch (mode) { case concurrent_normal: - service_concurrent_normal_cycle(cause); + if (heap->mode()->is_generational()) { + // TODO: Only young collections for now. + // We'll add old collections later. + service_concurrent_normal_cycle(cause, heap->young_generation()); + } else { + service_concurrent_normal_cycle(cause, heap->global_generation()); + } break; case stw_degenerated: service_stw_degenerated_cycle(cause, degen_point); @@ -346,7 +354,7 @@ bool ShenandoahControlThread::check_soft_max_changed() const { return false; } -void ShenandoahControlThread::service_concurrent_normal_cycle(GCCause::Cause cause) { +void ShenandoahControlThread::service_concurrent_normal_cycle(GCCause::Cause cause, ShenandoahGeneration* generation) { // Normal cycle goes via all concurrent phases. If allocation failure (af) happens during // any of the concurrent phases, it first degrades to Degenerated GC and completes GC there. // If second allocation failure happens during Degenerated GC cycle (for example, when GC @@ -395,14 +403,14 @@ void ShenandoahControlThread::service_concurrent_normal_cycle(GCCause::Cause cau heap->entry_reset(); // Start initial mark under STW - heap->vmop_entry_init_mark(); + heap->vmop_entry_init_mark(generation); // Continue concurrent mark - heap->entry_mark(); + heap->entry_mark(generation); if (check_cancellation_or_degen(ShenandoahHeap::_degenerated_mark)) return; // Complete marking under STW, and start evacuation - heap->vmop_entry_final_mark(); + heap->vmop_entry_final_mark(generation); // Process weak roots that might still point to regions that would be broken by cleanup if (heap->is_concurrent_weak_root_in_progress()) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp index 634194a170b4e..4e2e3ef93f04a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp @@ -100,7 +100,7 @@ class ShenandoahControlThread: public ConcurrentGCThread { shenandoah_padding(2); bool check_cancellation_or_degen(ShenandoahHeap::ShenandoahDegenPoint point); - void service_concurrent_normal_cycle(GCCause::Cause cause); + void service_concurrent_normal_cycle(GCCause::Cause cause, ShenandoahGeneration* generation); void service_stw_full_cycle(GCCause::Cause cause); void service_stw_degenerated_cycle(GCCause::Cause cause, ShenandoahHeap::ShenandoahDegenPoint point); void service_uncommit(double shrink_before, size_t shrink_until); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index 4bafad82939b5..b336f7d3d35fc 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -146,14 +146,14 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah return NULL; } + try_recycle_trashed(r); + if (r->affiliation() == FREE) { r->set_affiliation(req.affiliation()); } else if (r->affiliation() != req.affiliation()) { return NULL; } - try_recycle_trashed(r); - in_new_region = r->is_empty(); HeapWord* result = NULL; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index e016bf1e9c922..be41bec1344be 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -25,15 +25,22 @@ #ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHGENERATION_HPP #define SHARE_VM_GC_SHENANDOAH_SHENANDOAHGENERATION_HPP -class ShenandoahGeneration { +#include "memory/allocation.hpp" +#include "gc/shenandoah/shenandoahConcurrentMark.hpp" + +class ShenandoahGeneration : public CHeapObj { +private: + GenerationMode const _generation_mode; + ShenandoahConcurrentMark* const _scm; public: - ShenandoahGeneration(); -}; + ShenandoahGeneration(GenerationMode generation_mode) : + _generation_mode(generation_mode), + _scm(new ShenandoahConcurrentMark(generation_mode)) { + } -class ShenandoahYoungGeneration : public ShenandoahGeneration { -}; + inline GenerationMode generation_mode() const { return _generation_mode; } -class ShenandoahOldGeneration : public ShenandoahGeneration { + inline ShenandoahConcurrentMark* concurrent_mark() const { return _scm; } }; #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHGENERATION_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp new file mode 100644 index 0000000000000..224c5f25d7bf2 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2020, Amazon.com, Inc. and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHGLOBALGENERATION_HPP +#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHGLOBALGENERATION_HPP + +#include "gc/shenandoah/shenandoahGeneration.hpp" + +// A "generation" that represents the whole heap. +class ShenandoahGlobalGeneration : public ShenandoahGeneration { +public: + ShenandoahGlobalGeneration() : ShenandoahGeneration(GLOBAL) { } +}; + +#endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHGLOBALGENERATION_HPP + diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 385aafef762ec..881e5b90ded9d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -42,6 +42,7 @@ #include "gc/shenandoah/shenandoahConcurrentRoots.hpp" #include "gc/shenandoah/shenandoahControlThread.hpp" #include "gc/shenandoah/shenandoahFreeSet.hpp" +#include "gc/shenandoah/shenandoahGlobalGeneration.hpp" #include "gc/shenandoah/shenandoahPhaseTimings.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" @@ -66,6 +67,7 @@ #include "gc/shenandoah/shenandoahVMOperations.hpp" #include "gc/shenandoah/shenandoahWorkGroup.hpp" #include "gc/shenandoah/shenandoahWorkerPolicy.hpp" +#include "gc/shenandoah/shenandoahYoungGeneration.hpp" #include "gc/shenandoah/mode/shenandoahGenerationalMode.hpp" #include "gc/shenandoah/mode/shenandoahIUMode.hpp" #include "gc/shenandoah/mode/shenandoahPassiveMode.hpp" @@ -484,11 +486,12 @@ ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) : _num_regions(0), _regions(NULL), _update_refs_iterator(this), + _young_generation(new ShenandoahYoungGeneration()), + _global_generation(new ShenandoahGlobalGeneration()), _control_thread(NULL), _shenandoah_policy(policy), _heuristics(NULL), _free_set(NULL), - _scm(new ShenandoahConcurrentMark()), _full_gc(new ShenandoahMarkCompact()), _pacer(NULL), _verifier(NULL), @@ -618,7 +621,8 @@ void ShenandoahHeap::post_initialize() { // Now, we will let WorkGang to initialize gclab when new worker is created. _workers->set_initialize_gclab(); - _scm->initialize(_max_workers); + young_generation()->concurrent_mark()->initialize(_max_workers); + global_generation()->concurrent_mark()->initialize(_max_workers); _full_gc->initialize(_gc_timer); _heuristics->initialize(); @@ -1583,7 +1587,7 @@ class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionCl bool is_thread_safe() { return true; } }; -void ShenandoahHeap::op_init_mark() { +void ShenandoahHeap::op_init_mark(ShenandoahGeneration* generation) { assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint"); assert(Thread::current()->is_VM_thread(), "can only do this in VMThread"); @@ -1618,7 +1622,7 @@ void ShenandoahHeap::op_init_mark() { // Make above changes visible to worker threads OrderAccess::fence(); - concurrent_mark()->mark_roots(ShenandoahPhaseTimings::scan_roots); + generation->concurrent_mark()->mark_roots(ShenandoahPhaseTimings::scan_roots); if (ShenandoahPacing) { pacer()->setup_for_mark(); @@ -1632,8 +1636,8 @@ void ShenandoahHeap::op_init_mark() { } } -void ShenandoahHeap::op_mark() { - concurrent_mark()->mark_from_roots(); +void ShenandoahHeap::op_mark(ShenandoahGeneration* generation) { + generation->concurrent_mark()->mark_from_roots(); } class ShenandoahFinalMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure { @@ -1683,7 +1687,7 @@ class ShenandoahFinalMarkUpdateRegionStateClosure : public ShenandoahHeapRegionC bool is_thread_safe() { return true; } }; -void ShenandoahHeap::op_final_mark() { +void ShenandoahHeap::op_final_mark(ShenandoahGeneration* generation) { assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint"); assert(!has_forwarded_objects(), "No forwarded objects on this path"); @@ -1692,7 +1696,7 @@ void ShenandoahHeap::op_final_mark() { // get unmarked objects in the roots. if (!cancelled_gc()) { - concurrent_mark()->finish_mark_from_roots(/* full_gc = */ false); + generation->concurrent_mark()->finish_mark_from_roots(/* full_gc = */ false); // Marking is completed, deactivate SATB barrier set_concurrent_mark_in_progress(false); @@ -1792,7 +1796,7 @@ void ShenandoahHeap::op_final_mark() { } else { // If this cycle was updating references, we need to keep the has_forwarded_objects // flag on, for subsequent phases to deal with it. - concurrent_mark()->cancel(); + generation->concurrent_mark()->cancel(); set_concurrent_mark_in_progress(false); // Abandon reference processing right away: pre-cleaning must have failed. @@ -2184,14 +2188,14 @@ void ShenandoahHeap::op_degenerated(ShenandoahDegenPoint point) { op_reset(); - op_init_mark(); + op_init_mark(ShenandoahHeap::heap()->global_generation()); if (cancelled_gc()) { op_degenerated_fail(); return; } case _degenerated_mark: - op_final_mark(); + op_final_mark(ShenandoahHeap::heap()->global_generation()); if (cancelled_gc()) { op_degenerated_fail(); return; @@ -2793,7 +2797,7 @@ void ShenandoahHeap::op_final_updaterefs() { } if (is_degenerated_gc_in_progress()) { - concurrent_mark()->update_roots(ShenandoahPhaseTimings::degen_gc_update_roots); + global_generation()->concurrent_mark()->update_roots(ShenandoahPhaseTimings::degen_gc_update_roots); } // Has to be done before cset is clear @@ -2919,21 +2923,21 @@ void ShenandoahHeap::safepoint_synchronize_end() { } } -void ShenandoahHeap::vmop_entry_init_mark() { +void ShenandoahHeap::vmop_entry_init_mark(ShenandoahGeneration* generation) { TraceCollectorStats tcs(monitoring_support()->stw_collection_counters()); ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_mark_gross); try_inject_alloc_failure(); - VM_ShenandoahInitMark op; + VM_ShenandoahInitMark op(generation); VMThread::execute(&op); // jump to entry_init_mark() under safepoint } -void ShenandoahHeap::vmop_entry_final_mark() { +void ShenandoahHeap::vmop_entry_final_mark(ShenandoahGeneration* generation) { TraceCollectorStats tcs(monitoring_support()->stw_collection_counters()); ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_mark_gross); try_inject_alloc_failure(); - VM_ShenandoahFinalMarkStartEvac op; + VM_ShenandoahFinalMarkStartEvac op(generation); VMThread::execute(&op); // jump to entry_final_mark under safepoint } @@ -2972,7 +2976,7 @@ void ShenandoahHeap::vmop_degenerated(ShenandoahDegenPoint point) { VMThread::execute(°enerated_gc); } -void ShenandoahHeap::entry_init_mark() { +void ShenandoahHeap::entry_init_mark(ShenandoahGeneration* generation) { const char* msg = init_mark_event_message(); ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_mark); EventMark em("%s", msg); @@ -2981,10 +2985,10 @@ void ShenandoahHeap::entry_init_mark() { ShenandoahWorkerPolicy::calc_workers_for_init_marking(), "init marking"); - op_init_mark(); + op_init_mark(generation); } -void ShenandoahHeap::entry_final_mark() { +void ShenandoahHeap::entry_final_mark(ShenandoahGeneration* generation) { const char* msg = final_mark_event_message(); ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_mark); EventMark em("%s", msg); @@ -2993,7 +2997,7 @@ void ShenandoahHeap::entry_final_mark() { ShenandoahWorkerPolicy::calc_workers_for_final_marking(), "final marking"); - op_final_mark(); + op_final_mark(generation); } void ShenandoahHeap::entry_init_updaterefs() { @@ -3045,7 +3049,7 @@ void ShenandoahHeap::entry_degenerated(int point) { set_degenerated_gc_in_progress(false); } -void ShenandoahHeap::entry_mark() { +void ShenandoahHeap::entry_mark(ShenandoahGeneration* generation) { TraceCollectorStats tcs(monitoring_support()->concurrent_collection_counters()); const char* msg = conc_mark_event_message(); @@ -3057,7 +3061,7 @@ void ShenandoahHeap::entry_mark() { "concurrent marking"); try_inject_alloc_failure(); - op_mark(); + op_mark(generation); } void ShenandoahHeap::entry_evac() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index 6c3617bcc329b..b364f82dc09da 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -46,6 +46,7 @@ class ShenandoahCollectorPolicy; class ShenandoahControlThread; class ShenandoahGCSession; class ShenandoahGCStateResetter; +class ShenandoahGeneration; class ShenandoahHeuristics; class ShenandoahMarkingContext; class ShenandoahMarkCompact; @@ -57,7 +58,6 @@ class ShenandoahHeapRegionClosure; class ShenandoahCollectionSet; class ShenandoahFreeSet; class ShenandoahConcurrentMark; -class ShenandoahMarkCompact; class ShenandoahMonitoringSupport; class ShenandoahReferenceProcessor; class ShenandoahPacer; @@ -370,8 +370,8 @@ class ShenandoahHeap : public CollectedHeap { public: // Entry points to STW GC operations, these cause a related safepoint, that then // call the entry method below - void vmop_entry_init_mark(); - void vmop_entry_final_mark(); + void vmop_entry_init_mark(ShenandoahGeneration* generation); + void vmop_entry_final_mark(ShenandoahGeneration* generation); void vmop_entry_init_updaterefs(); void vmop_entry_final_updaterefs(); void vmop_entry_full(GCCause::Cause cause); @@ -379,8 +379,8 @@ class ShenandoahHeap : public CollectedHeap { // Entry methods to normally STW GC operations. These set up logging, monitoring // and workers for net VM operation - void entry_init_mark(); - void entry_final_mark(); + void entry_init_mark(ShenandoahGeneration* generation); + void entry_final_mark(ShenandoahGeneration* generation); void entry_init_updaterefs(); void entry_final_updaterefs(); void entry_full(GCCause::Cause cause); @@ -389,7 +389,7 @@ class ShenandoahHeap : public CollectedHeap { // Entry methods to normally concurrent GC operations. These set up logging, monitoring // for concurrent operation. void entry_reset(); - void entry_mark(); + void entry_mark(ShenandoahGeneration* generation); void entry_weak_refs(); void entry_weak_roots(); void entry_class_unloading(); @@ -404,8 +404,8 @@ class ShenandoahHeap : public CollectedHeap { private: // Actual work for the phases - void op_init_mark(); - void op_final_mark(); + void op_init_mark(ShenandoahGeneration* generation); + void op_final_mark(ShenandoahGeneration* generation); void op_init_updaterefs(); void op_final_updaterefs(); void op_full(GCCause::Cause cause); @@ -414,7 +414,7 @@ class ShenandoahHeap : public CollectedHeap { void op_degenerated_futile(); void op_reset(); - void op_mark(); + void op_mark(ShenandoahGeneration* generation); void op_weak_refs(); void op_weak_roots(); void op_class_unloading(); @@ -440,12 +440,13 @@ class ShenandoahHeap : public CollectedHeap { // ---------- GC subsystems // private: + ShenandoahGeneration* _young_generation; + ShenandoahGeneration* _global_generation; ShenandoahControlThread* _control_thread; ShenandoahCollectorPolicy* _shenandoah_policy; ShenandoahMode* _gc_mode; ShenandoahHeuristics* _heuristics; ShenandoahFreeSet* _free_set; - ShenandoahConcurrentMark* _scm; ShenandoahMarkCompact* _full_gc; ShenandoahPacer* _pacer; ShenandoahVerifier* _verifier; @@ -456,11 +457,12 @@ class ShenandoahHeap : public CollectedHeap { ShenandoahMarkCompact* full_gc() { return _full_gc; } public: + ShenandoahGeneration* young_generation() const { return _young_generation; } + ShenandoahGeneration* global_generation() const { return _global_generation; } ShenandoahCollectorPolicy* shenandoah_policy() const { return _shenandoah_policy; } ShenandoahMode* mode() const { return _gc_mode; } ShenandoahHeuristics* heuristics() const { return _heuristics; } ShenandoahFreeSet* free_set() const { return _free_set; } - ShenandoahConcurrentMark* concurrent_mark() { return _scm; } ShenandoahPacer* pacer() const { return _pacer; } ShenandoahPhaseTimings* phase_timings() const { return _phase_timings; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp index 846b42c73ad2b..a1f455f61898d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkCompact.cpp @@ -32,6 +32,7 @@ #include "gc/shenandoah/shenandoahConcurrentRoots.hpp" #include "gc/shenandoah/shenandoahCollectionSet.hpp" #include "gc/shenandoah/shenandoahFreeSet.hpp" +#include "gc/shenandoah/shenandoahGeneration.hpp" #include "gc/shenandoah/shenandoahPhaseTimings.hpp" #include "gc/shenandoah/shenandoahMarkCompact.hpp" #include "gc/shenandoah/shenandoahHeapRegionSet.hpp" @@ -114,14 +115,14 @@ void ShenandoahMarkCompact::do_it(GCCause::Cause gc_cause) { // b. Cancel concurrent mark, if in progress if (heap->is_concurrent_mark_in_progress()) { - heap->concurrent_mark()->cancel(); + heap->global_generation()->concurrent_mark()->cancel(); heap->set_concurrent_mark_in_progress(false); } assert(!heap->is_concurrent_mark_in_progress(), "sanity"); // c. Update roots if this full GC is due to evac-oom, which may carry from-space pointers in roots. if (has_forwarded_objects) { - heap->concurrent_mark()->update_roots(ShenandoahPhaseTimings::full_gc_update_roots); + heap->global_generation()->concurrent_mark()->update_roots(ShenandoahPhaseTimings::full_gc_update_roots); } // d. Reset the bitmaps for new marking @@ -238,7 +239,7 @@ void ShenandoahMarkCompact::phase1_mark_heap() { ShenandoahPrepareForMarkClosure cl; heap->heap_region_iterate(&cl); - ShenandoahConcurrentMark* cm = heap->concurrent_mark(); + ShenandoahConcurrentMark* cm = heap->global_generation()->concurrent_mark(); heap->set_unload_classes(heap->heuristics()->can_unload_classes()); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp index e9dceaad0717d..53ff0aed7f2f4 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp @@ -32,6 +32,11 @@ #include "memory/iterator.hpp" #include "runtime/thread.hpp" +enum GenerationMode { + YOUNG, + GLOBAL +}; + enum UpdateRefsMode { NONE, // No reference updating RESOLVE, // Only a resolve (no reference updating) @@ -52,7 +57,7 @@ class ShenandoahMarkRefsSuperClosure : public MetadataVisitingOopIterateClosure bool _weak; protected: - template + template void work(T *p); public: @@ -67,10 +72,11 @@ class ShenandoahMarkRefsSuperClosure : public MetadataVisitingOopIterateClosure } }; +template class ShenandoahMarkUpdateRefsClosure : public ShenandoahMarkRefsSuperClosure { private: template - inline void do_oop_work(T* p) { work(p); } + inline void do_oop_work(T* p) { work(p); } public: ShenandoahMarkUpdateRefsClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) : @@ -81,10 +87,11 @@ class ShenandoahMarkUpdateRefsClosure : public ShenandoahMarkRefsSuperClosure { virtual bool do_metadata() { return false; } }; +template class ShenandoahMarkUpdateRefsDedupClosure : public ShenandoahMarkRefsSuperClosure { private: template - inline void do_oop_work(T* p) { work(p); } + inline void do_oop_work(T* p) { work(p); } public: ShenandoahMarkUpdateRefsDedupClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) : @@ -95,10 +102,11 @@ class ShenandoahMarkUpdateRefsDedupClosure : public ShenandoahMarkRefsSuperClosu virtual bool do_metadata() { return false; } }; +template class ShenandoahMarkUpdateRefsMetadataClosure : public ShenandoahMarkRefsSuperClosure { private: template - inline void do_oop_work(T* p) { work(p); } + inline void do_oop_work(T* p) { work(p); } public: ShenandoahMarkUpdateRefsMetadataClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) : @@ -109,10 +117,11 @@ class ShenandoahMarkUpdateRefsMetadataClosure : public ShenandoahMarkRefsSuperCl virtual bool do_metadata() { return true; } }; +template class ShenandoahMarkUpdateRefsMetadataDedupClosure : public ShenandoahMarkRefsSuperClosure { private: template - inline void do_oop_work(T* p) { work(p); } + inline void do_oop_work(T* p) { work(p); } public: ShenandoahMarkUpdateRefsMetadataDedupClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) : @@ -123,10 +132,11 @@ class ShenandoahMarkUpdateRefsMetadataDedupClosure : public ShenandoahMarkRefsSu virtual bool do_metadata() { return true; } }; +template class ShenandoahMarkRefsClosure : public ShenandoahMarkRefsSuperClosure { private: template - inline void do_oop_work(T* p) { work(p); } + inline void do_oop_work(T* p) { work(p); } public: ShenandoahMarkRefsClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) : @@ -137,10 +147,11 @@ class ShenandoahMarkRefsClosure : public ShenandoahMarkRefsSuperClosure { virtual bool do_metadata() { return false; } }; +template class ShenandoahMarkRefsDedupClosure : public ShenandoahMarkRefsSuperClosure { private: template - inline void do_oop_work(T* p) { work(p); } + inline void do_oop_work(T* p) { work(p); } public: ShenandoahMarkRefsDedupClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) : @@ -151,10 +162,11 @@ class ShenandoahMarkRefsDedupClosure : public ShenandoahMarkRefsSuperClosure { virtual bool do_metadata() { return false; } }; +template class ShenandoahMarkResolveRefsClosure : public ShenandoahMarkRefsSuperClosure { private: template - inline void do_oop_work(T* p) { work(p); } + inline void do_oop_work(T* p) { work(p); } public: ShenandoahMarkResolveRefsClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) : @@ -165,10 +177,11 @@ class ShenandoahMarkResolveRefsClosure : public ShenandoahMarkRefsSuperClosure { virtual bool do_metadata() { return false; } }; +template class ShenandoahMarkRefsMetadataClosure : public ShenandoahMarkRefsSuperClosure { private: template - inline void do_oop_work(T* p) { work(p); } + inline void do_oop_work(T* p) { work(p); } public: ShenandoahMarkRefsMetadataClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) : @@ -179,10 +192,11 @@ class ShenandoahMarkRefsMetadataClosure : public ShenandoahMarkRefsSuperClosure virtual bool do_metadata() { return true; } }; +template class ShenandoahMarkRefsMetadataDedupClosure : public ShenandoahMarkRefsSuperClosure { private: template - inline void do_oop_work(T* p) { work(p); } + inline void do_oop_work(T* p) { work(p); } public: ShenandoahMarkRefsMetadataDedupClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) : diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp index 3476f65a7fe1e..f9de4067e14c7 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp @@ -27,10 +27,11 @@ #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahConcurrentMark.inline.hpp" +#include "gc/shenandoah/shenandoahGeneration.hpp" -template +template inline void ShenandoahMarkRefsSuperClosure::work(T *p) { - ShenandoahConcurrentMark::mark_through_ref(p, _heap, _queue, _mark_context, _weak); + ShenandoahConcurrentMark::mark_through_ref(p, _heap, _queue, _mark_context, _weak); } template diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp index 0fd3bc9d9c960..e243cc9688d9d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp @@ -25,6 +25,7 @@ #include "precompiled.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" +#include "gc/shenandoah/shenandoahGeneration.hpp" #include "gc/shenandoah/shenandoahUtils.hpp" #include "gc/shenandoah/shenandoahVMOperations.hpp" #include "memory/universe.hpp" @@ -43,12 +44,12 @@ void VM_ShenandoahReferenceOperation::doit_epilogue() { void VM_ShenandoahInitMark::doit() { ShenandoahGCPauseMark mark(_gc_id, SvcGCMarker::CONCURRENT); - ShenandoahHeap::heap()->entry_init_mark(); + ShenandoahHeap::heap()->entry_init_mark(_generation); } void VM_ShenandoahFinalMarkStartEvac::doit() { ShenandoahGCPauseMark mark(_gc_id, SvcGCMarker::CONCURRENT); - ShenandoahHeap::heap()->entry_final_mark(); + ShenandoahHeap::heap()->entry_final_mark(_generation); } void VM_ShenandoahFullGC::doit() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp index b3a4aef6426d3..8fa05ec3bfa5e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp @@ -52,16 +52,20 @@ class VM_ShenandoahReferenceOperation : public VM_ShenandoahOperation { }; class VM_ShenandoahInitMark: public VM_ShenandoahOperation { +private: + ShenandoahGeneration* _generation; public: - VM_ShenandoahInitMark() : VM_ShenandoahOperation() {}; + VM_ShenandoahInitMark(ShenandoahGeneration* generation) : VM_ShenandoahOperation(), _generation(generation) {}; VM_Operation::VMOp_Type type() const { return VMOp_ShenandoahInitMark; } const char* name() const { return "Shenandoah Init Marking"; } virtual void doit(); }; class VM_ShenandoahFinalMarkStartEvac: public VM_ShenandoahOperation { +private: + ShenandoahGeneration* _generation; public: - VM_ShenandoahFinalMarkStartEvac() : VM_ShenandoahOperation() {}; + VM_ShenandoahFinalMarkStartEvac(ShenandoahGeneration* generation) : VM_ShenandoahOperation(), _generation(generation) {}; VM_Operation::VMOp_Type type() const { return VMOp_ShenandoahFinalMarkStartEvac; } const char* name() const { return "Shenandoah Final Mark and Start Evacuation"; } virtual void doit(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp similarity index 77% rename from src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp rename to src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp index 24c87caa50182..588c630d7f5fc 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp @@ -22,5 +22,14 @@ * */ -#include "precompiled.hpp" +#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHYOUNGGENERATION_HPP +#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHYOUNGGENERATION_HPP + #include "gc/shenandoah/shenandoahGeneration.hpp" + +class ShenandoahYoungGeneration : public ShenandoahGeneration { +public: + ShenandoahYoungGeneration() : ShenandoahGeneration(YOUNG) { } +}; + +#endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHYOUNGGENERATION_HPP From 9358d48bd3ce2401cb411d9aead9f622bfe7e6e4 Mon Sep 17 00:00:00 2001 From: Bernd Mathiske Date: Mon, 14 Dec 2020 15:42:03 +0000 Subject: [PATCH 007/254] Marking and evacuation constrained to young gen in generational mode. Reviewed-by: rkennke --- .../heuristics/shenandoahHeuristics.cpp | 3 ++ .../shenandoahConcurrentMark.inline.hpp | 30 ++++++++++--------- .../share/gc/shenandoah/shenandoahFreeSet.cpp | 1 + 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp index 66e7bca9ce67d..009cf11034d40 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp @@ -96,6 +96,9 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec for (size_t i = 0; i < num_regions; i++) { ShenandoahHeapRegion* region = heap->get_region(i); + if (heap->mode()->is_generational() && region->affiliation() != ShenandoahRegionAffiliation::YOUNG_GENERATION) { + continue; + } size_t garbage = region->garbage(); total_garbage += garbage; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.inline.hpp index ecf3bfc8f4cfc..9340143a67420 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.inline.hpp @@ -266,20 +266,22 @@ inline void ShenandoahConcurrentMark::mark_through_ref(T *p, ShenandoahHeap* hea shenandoah_assert_not_forwarded(p, obj); shenandoah_assert_not_in_cset_except(p, obj, heap->cancelled_gc()); - bool skip_live = false; - bool marked; - if (weak) { - marked = mark_context->mark_weak(obj); - } else { - marked = mark_context->mark_strong(obj, /* was_upgraded = */ skip_live); - } - if (marked) { - bool pushed = q->push(ShenandoahMarkTask(obj, skip_live, weak)); - assert(pushed, "overflow queue should always succeed pushing"); - - if ((STRING_DEDUP == ENQUEUE_DEDUP) && ShenandoahStringDedup::is_candidate(obj)) { - assert(ShenandoahStringDedup::is_enabled(), "Must be enabled"); - ShenandoahStringDedup::enqueue_candidate(obj); + if (GENERATION != YOUNG || heap->is_in_young(obj)) { + bool skip_live = false; + bool marked; + if (weak) { + marked = mark_context->mark_weak(obj); + } else { + marked = mark_context->mark_strong(obj, /* was_upgraded = */ skip_live); + } + if (marked) { + bool pushed = q->push(ShenandoahMarkTask(obj, skip_live, weak)); + assert(pushed, "overflow queue should always succeed pushing"); + + if ((STRING_DEDUP == ENQUEUE_DEDUP) && ShenandoahStringDedup::is_candidate(obj)) { + assert(ShenandoahStringDedup::is_enabled(), "Must be enabled"); + ShenandoahStringDedup::enqueue_candidate(obj); + } } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index b336f7d3d35fc..7031174be5168 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -311,6 +311,7 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req) { } r->set_top(r->bottom() + used_words); + r->set_affiliation(req.affiliation()); _mutator_free_bitmap.clear_bit(r->index()); } From 8a38a9ede26832355e44597b98694abddad828af Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Fri, 15 Jan 2021 18:09:58 +0000 Subject: [PATCH 008/254] Scan remembered Reviewed-by: rkennke, zgu --- .../shenandoah/shenandoahConcurrentMark.cpp | 38 +- .../share/gc/shenandoah/shenandoahHeap.cpp | 70 +- .../share/gc/shenandoah/shenandoahHeap.hpp | 10 + .../gc/shenandoah/shenandoahHeap.inline.hpp | 8 +- .../shenandoahReferenceProcessor.hpp | 3 +- .../shenandoah/shenandoahScanRemembered.cpp | 86 ++ .../shenandoah/shenandoahScanRemembered.hpp | 934 ++++++++++++++++++ .../shenandoahScanRemembered.inline.hpp | 399 ++++++++ .../gc/shenandoah/shenandoahStringDedup.cpp | 19 +- 9 files changed, 1541 insertions(+), 26 deletions(-) create mode 100644 src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp create mode 100644 src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp create mode 100644 src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp index f4ad0dcfaeb92..cee02e4161bc3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp @@ -44,6 +44,7 @@ #include "gc/shenandoah/shenandoahPhaseTimings.hpp" #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp" #include "gc/shenandoah/shenandoahUtils.hpp" +#include "gc/shenandoah/shenandoahScanRemembered.inline.hpp" #include "memory/iterator.inline.hpp" #include "memory/metaspace.hpp" @@ -86,11 +87,14 @@ class ShenandoahInitMarkRootsTask : public AbstractGangTask { private: ShenandoahConcurrentMark* const _scm; ShenandoahRootScanner* const _rp; + uint const _workers; public: - ShenandoahInitMarkRootsTask(ShenandoahConcurrentMark* scm, ShenandoahRootScanner* rp) : + + ShenandoahInitMarkRootsTask(ShenandoahConcurrentMark* scm, ShenandoahRootScanner* rp, uint worker_count) : AbstractGangTask("Shenandoah Init Mark Roots"), _scm(scm), - _rp(rp) { + _rp(rp), + _workers(worker_count) { } void work(uint worker_id) { @@ -106,6 +110,32 @@ class ShenandoahInitMarkRootsTask : public AbstractGangTask { switch (_scm->generation_mode()) { case YOUNG: { ShenandoahInitMarkRootsClosure mark_cl(q); + + // Do the remembered set scanning before the root scanning as the current implementation of remembered set scanning + // does not do workload balancing. If certain worker threads end up with disproportionate amounts of remembered set + // scanning effort, the subsequent root scanning effort will balance workload to even effort between threads. + uint32_t r; + RememberedScanner *rs = heap->card_scan(); + ShenandoahReferenceProcessor* rp = heap->ref_processor(); + unsigned int total_regions = heap->num_regions(); + + for (r = worker_id % _workers; r < total_regions; r += _workers) { + ShenandoahHeapRegion *region = heap->get_region(r); + if (region->affiliation() == OLD_GENERATION) { + uint32_t start_cluster_no = rs->cluster_for_addr(region->bottom()); + + // region->end() represents the end of memory spanned by this region, but not all of this + // memory is eligible to be scanned because some of this memory has not yet been allocated. + // + // region->top() represents the end of allocated memory within this region. Any addresses + // beyond region->top() should not be scanned as that memory does not hold valid objects. + HeapWord *end_of_range = region->top(); + uint32_t stop_cluster_no = rs->cluster_for_addr(end_of_range); + rs->process_clusters>(worker_id, rp, _scm, start_cluster_no, + stop_cluster_no + 1 - start_cluster_no, + end_of_range, &mark_cl); + } + } do_work(heap, &mark_cl, worker_id); break; } @@ -320,12 +350,12 @@ void ShenandoahConcurrentMark::mark_roots(ShenandoahPhaseTimings::Phase root_pha task_queues()->reserve(nworkers); if (heap->has_forwarded_objects()) { - ShenandoahInitMarkRootsTask mark_roots(this, &root_proc); + ShenandoahInitMarkRootsTask mark_roots(this, &root_proc, nworkers); workers->run_task(&mark_roots); } else { // No need to update references, which means the heap is stable. // Can save time not walking through forwarding pointers. - ShenandoahInitMarkRootsTask mark_roots(this, &root_proc); + ShenandoahInitMarkRootsTask mark_roots(this, &root_proc, nworkers); workers->run_task(&mark_roots); } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 881e5b90ded9d..4908751c18609 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -59,6 +59,7 @@ #include "gc/shenandoah/shenandoahParallelCleaning.inline.hpp" #include "gc/shenandoah/shenandoahReferenceProcessor.hpp" #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp" +#include "gc/shenandoah/shenandoahScanRemembered.inline.hpp" #include "gc/shenandoah/shenandoahStringDedup.hpp" #include "gc/shenandoah/shenandoahTaskqueue.hpp" #include "gc/shenandoah/shenandoahUtils.hpp" @@ -72,6 +73,7 @@ #include "gc/shenandoah/mode/shenandoahIUMode.hpp" #include "gc/shenandoah/mode/shenandoahPassiveMode.hpp" #include "gc/shenandoah/mode/shenandoahSATBMode.hpp" + #if INCLUDE_JFR #include "gc/shenandoah/shenandoahJfrSupport.hpp" #endif @@ -211,6 +213,12 @@ jint ShenandoahHeap::initialize() { // // After reserving the Java heap, create the card table, barriers, and workers, in dependency order // + if (mode()->is_generational()) { + ShenandoahDirectCardMarkRememberedSet *rs; + size_t card_count = ShenandoahBarrierSet::barrier_set()->card_table()->cards_required(heap_rs.size() / HeapWordSize) - 1; + rs = new ShenandoahDirectCardMarkRememberedSet(ShenandoahBarrierSet::barrier_set()->card_table(), card_count); + _card_scan = new ShenandoahScanRemembered(rs); + } BarrierSet::set_barrier_set(new ShenandoahBarrierSet(this, _heap_region)); _workers = new ShenandoahWorkGang("Shenandoah GC Threads", _max_workers, @@ -511,7 +519,8 @@ ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) : _bitmap_region_special(false), _aux_bitmap_region_special(false), _liveness_cache(NULL), - _collection_set(NULL) + _collection_set(NULL), + _card_scan(NULL) { } @@ -2680,21 +2689,72 @@ class ShenandoahUpdateHeapRefsTask : public AbstractGangTask { if (_concurrent) { ShenandoahConcurrentWorkerSession worker_session(worker_id); ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers); - do_work(); + do_work(worker_id); } else { ShenandoahParallelWorkerSession worker_session(worker_id); - do_work(); + do_work(worker_id); } } private: - void do_work() { + void do_work(uint worker_id) { ShenandoahHeapRegion* r = _regions->next(); ShenandoahMarkingContext* const ctx = _heap->complete_marking_context(); + while (r != NULL) { HeapWord* update_watermark = r->get_update_watermark(); assert (update_watermark >= r->bottom(), "sanity"); - if (r->is_active() && !r->is_cset()) { + + // Eventually, scanning of old-gen memory regions for the purpose of updating references can happen + // concurrently. This can be done during concurrent evacuation of roots for example. + if (r->is_active() && (r->affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) && !r->is_cset()) { + + // Note that we use this code even if we are doing an old-gen collection and we have a bitmap to + // represent marked objects within the heap region. + // + // It is necessary to process all objects rather than just the marked objects during update-refs of + // an old-gen region as part of an old-gen collection. Otherwise, a subseqent update-refs scan of + // the same region will see stale pointers and crash. + // + // r->top() represents the upper end of memory that has been allocated within this region. + // As new objects are allocated, the value of r->top() increases to accomodate each new + // object. + // At the start of evacuation, "update_watermark" is initalized to represent the value of top(). + // Objects newly allocated during evacuation do not need to be visited during update-refs + // because the to-space invariant which is in force throughout evacuation assures that no from-space + // pointer is written to any newly allocated object. In the case that survivor objects are evacuated + // into this region during evacuation, the region's watermark is incremented to represent the end of + // the memory range known to hold newly evacuated objects. Regions that receive evacuated objects + // are distinct from regions that serve new object allocation requests. A region's watermark is not + // increased when objects are newly allocated within that region during evacuation. + + HeapWord *p = r->bottom(); + ShenandoahObjectToOopBoundedClosure objs(&cl, p, update_watermark); + + // TODO: This code assumes every object ever allocated within this old-gen region is still live. If we + // allow a sweep phase to turn garbage objects into free memory regions, we need to modify this code to + // skip over and/or synchronize access to these free memory regions. There might be races, for example, + // if we are trying to scan one of these free memory regions while a different thread is trying to + // allocate from within a free region. + // + // Alternative approaches are also under consideration. For example: + // 1. Coalesce, fill, and register each range of contiguous dead objects so that subsequent updating of + // references can be done more efficiently. + // 2. Retain the mark bitmap from the most recently completed old GC effort and use this bitmap to allow + // skipping over objects that were not live as of the most recently completed old-gen GC effort. + + // Anything beyond update_watermark does not need to be updated. + while (p < update_watermark) { + oop obj = oop(p); + + // The invocation of do_object() is "borrowed" from the implementation of + // ShenandoahHeap::marked_object_iterate(), which is called by _heap->marked_object_oop_iterate(). + objs.do_object(obj); + p += obj->size(); + + } + } + else if (r->is_active() && !r->is_cset()) { _heap->marked_object_oop_iterate(r, &cl, update_watermark); } if (ShenandoahPacing) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index b364f82dc09da..ed74f52b8723b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -35,6 +35,7 @@ #include "gc/shenandoah/shenandoahPadding.hpp" #include "gc/shenandoah/shenandoahSharedVariables.hpp" #include "gc/shenandoah/shenandoahUnload.hpp" +#include "gc/shenandoah/shenandoahScanRemembered.hpp" #include "memory/metaspace.hpp" #include "services/memoryManager.hpp" #include "utilities/globalDefinitions.hpp" @@ -695,6 +696,15 @@ class ShenandoahHeap : public CollectedHeap { inline void enter_evacuation(Thread* t); inline void leave_evacuation(Thread* t); +// ---------- Generational support +// +private: + RememberedScanner* _card_scan; + +public: + inline RememberedScanner* card_scan() { return _card_scan; } + + // ---------- Helper functions // public: diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp index 3a3ebd9faff3d..9f3bc8001ce7f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp @@ -312,7 +312,13 @@ inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) { // Successfully evacuated. Our copy is now the public one! shenandoah_assert_correct(NULL, copy_val); - // Increment age in young copies + // Hey! This code showed up in a merge conflict. It has "nothing" to do with the patch that + // was merged, so kdnilsen is leaving it in place as is. However, it looks to me like the object's + // age should be incremented before the copy is committed to avoid the need for synchronization here. + // + // kdnilsen believes the following code is replaced/relocated in a subsequent commit. + + // Increment age in young copies. if (target_gen == YOUNG_GENERATION) { copy_val->incr_age(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.hpp b/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.hpp index 40dbbc37f6e1f..82dd65dece881 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.hpp @@ -26,7 +26,8 @@ #ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHREFERENCEPROCESSOR_HPP #define SHARE_VM_GC_SHENANDOAH_SHENANDOAHREFERENCEPROCESSOR_HPP -#include "gc/shared/referenceDiscoverer.hpp" +// #include "gc/shared/referenceDiscoverer.hpp" +// #include "gc/shared/referencePolicy.hpp" #include "memory/allocation.hpp" class ShenandoahMarkRefsSuperClosure; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp new file mode 100644 index 0000000000000..19fa8ce42df0e --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2021, Amazon.com, Inc. or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + + +#include "precompiled.hpp" +#include "gc/shenandoah/shenandoahHeapRegion.hpp" +#include "gc/shenandoah/shenandoahScanRemembered.hpp" +#include "gc/shenandoah/shenandoahHeap.hpp" +#include "gc/shenandoah/shenandoahHeap.inline.hpp" +#include "gc/shenandoah/shenandoahHeapRegion.hpp" +#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" + +ShenandoahDirectCardMarkRememberedSet::ShenandoahDirectCardMarkRememberedSet(CardTable *card_table, size_t total_card_count) { + _heap = ShenandoahHeap::heap(); + _card_table = card_table; + _total_card_count = total_card_count; + _cluster_count = (total_card_count / ShenandoahCardCluster::CardsPerCluster); + _card_shift = CardTable::card_shift; + + _byte_map = _card_table->byte_for_index(0); + + _whole_heap_base = _card_table->addr_for(_byte_map); + _whole_heap_end = _whole_heap_base + total_card_count * CardTable::card_size; + + _byte_map_base = _byte_map - (uintptr_t(_whole_heap_base) >> _card_shift); + + _overreach_map = (uint8_t *) malloc(total_card_count); + _overreach_map_base = (_overreach_map - + (uintptr_t(_whole_heap_base) >> _card_shift)); + + assert(total_card_count % ShenandoahCardCluster::CardsPerCluster == 0, "Invalid card count."); + assert(total_card_count > 0, "Card count cannot be zero."); + // assert(_overreach_cards != NULL); +} + +ShenandoahDirectCardMarkRememberedSet::~ShenandoahDirectCardMarkRememberedSet() { + free(_overreach_map); +} + +void ShenandoahDirectCardMarkRememberedSet::initialize_overreach(uint32_t first_cluster, uint32_t count) { + + // We can make this run faster in the future by explicitly + // unrolling the loop and doing wide writes if the compiler + // doesn't do this for us. + uint32_t first_card_index = first_cluster * ShenandoahCardCluster::CardsPerCluster; + uint8_t *omp = &_overreach_map[first_card_index]; + uint8_t *endp = omp + count * ShenandoahCardCluster::CardsPerCluster; + while (omp < endp) + *omp++ = CardTable::clean_card_val(); +} + +void ShenandoahDirectCardMarkRememberedSet::merge_overreach(uint32_t first_cluster, uint32_t count) { + + // We can make this run faster in the future by explicitly unrolling the loop and doing wide writes if the compiler + // doesn't do this for us. + uint32_t first_card_index = first_cluster * ShenandoahCardCluster::CardsPerCluster; + uint8_t *bmp = &_byte_map[first_card_index]; + uint8_t *endp = bmp + count * ShenandoahCardCluster::CardsPerCluster; + uint8_t *omp = &_overreach_map[first_card_index]; + + // dirty_card is 0, clean card is 0xff; if either *bmp or *omp is dirty, we need to mark it as dirty + while (bmp < endp) + *bmp++ &= *omp++; +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp new file mode 100644 index 0000000000000..aa3d0895ec681 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp @@ -0,0 +1,934 @@ +/* + * Copyright (c) 2021, Amazon.com, Inc. or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + + +#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHSCANREMEMBERED_HPP +#define SHARE_GC_SHENANDOAH_SHENANDOAHSCANREMEMBERED_HPP + +#include +#include "memory/iterator.hpp" +#include "gc/shenandoah/shenandoahCardTable.hpp" +#include "gc/shared/referenceDiscoverer.hpp" +#include "gc/shared/referencePolicy.hpp" +#include "gc/shenandoah/shenandoahReferenceProcessor.hpp" + +// Terminology used within this source file: +// +// Card Entry: This is the information that identifies whether a +// particular card-table entry is Clean or Dirty. A clean +// card entry denotes that the associated memory does not +// hold references to young-gen memory. +// +// Card Region, aka +// Card Memory: This is the region of memory that is assocated with a +// particular card entry. +// +// Card Cluster: A card cluster represents 64 card entries. A card +// cluster is the minimal amount of work performed at a +// time by a parallel thread. Note that the work required +// to scan a card cluster is somewhat variable in that the +// required effort depends on how many cards are dirty, how +// many references are held within the objects that span a +// DIRTY card's memory, and on the size of the object +// that spans the end of a DIRTY card's memory (because +// that object will be scanned in its entirety). For these +// reasons, it is advisable for the multiple worker threads +// to be flexible in the number of clusters to be +// processed by each thread. +// +// A cluster represents a "natural" quantum of work to be performed by +// a parallel GC thread's background remembered set scanning efforts. +// The notion of cluster is similar to the notion of stripe in the +// implementation of parallel GC card scanning. However, a cluster is +// typically smaller than a stripe, enabling finer grain division of +// labor between multiple threads. +// +// For illustration, consider the following possible JVM configurations: +// +// Scenario 1: +// RegionSize is 128 MB +// Span of a card entry is 512 B +// Each card table entry consumes 1 B +// Assume one long word of card table entries represents a cluster. +// This long word holds 8 card table entries, spanning a +// total of 4KB +// The number of clusters per region is 128 MB / 4 KB = 32K +// +// Scenario 2: +// RegionSize is 128 MB +// Span of each card entry is 128 B +// Each card table entry consumes 1 bit +// Assume one int word of card tables represents a cluster. +// This int word holds 32 card table entries, spanning a +// total of 4KB +// The number of clusters per region is 128 MB / 4 KB = 32K +// +// Scenario 3: +// RegionSize is 128 MB +// Span of each card entry is 512 B +// Each card table entry consumes 1 bit +// Assume one long word of card tables represents a cluster. +// This long word holds 64 card table entries, spanning a +// total of 32 KB +// The number of clusters per region is 128 MB / 32 KB = 4K +// +// At the start of a new young-gen concurrent mark pass, the gang of +// Shenandoah worker threads collaborate in performing the following +// actions: +// +// Let old_regions = number of ShenandoahHeapRegion comprising +// old-gen memory +// Let region_size = ShenandoahHeapRegion::region_size_bytes() +// represent the number of bytes in each region +// Let clusters_per_region = region_size / 512 +// Let rs represent the relevant RememberedSet implementation +// (an instance of ShenandoahDirectCardMarkRememberedSet or an instance +// of a to-be-implemented ShenandoahBufferWithSATBRememberedSet) +// +// for each ShenandoahHeapRegion old_region in the whole heap +// determine the cluster number of the first cluster belonging +// to that region +// for each cluster contained within that region +// Assure that exactly one worker thread initializes each +// cluster of overreach memory by invoking: +// +// rs->initialize_overreach(cluster_no, cluster_count) +// +// in separate threads. (Divide up the clusters so that +// different threads are responsible for initializing different +// clusters. Initialization cost is essentially identical for +// each cluster.) +// +// Next, we repeat the process for invocations of process_Clusters. +// for each ShenandoahHeapRegion old_region in the whole heap +// determine the cluster number of the first cluster belonging +// to that region +// for each cluster contained within that region +// Assure that exactly one worker thread processes each +// cluster, each thread making a series of invocations of the +// following: +// +// rs->process_clusters(worker_id, ReferenceProcessor *, +// ShenandoahConcurrentMark *, cluster_no, cluster_count, +// HeapWord *end_of_range, OopClosure *oops); +// +// For efficiency, divide up the clusters so that different threads +// are responsible for processing different clusters. Processing costs +// may vary greatly between clusters for the following reasons: +// +// a) some clusters contain mostly dirty cards and other +// clusters contain mostly clean cards +// b) some clusters contain mostly primitive data and other +// clusters contain mostly reference data +// c) some clusters are spanned by very large objects that +// begin in some other cluster. When a large object +// beginning in a preceding cluster spans large portions of +// this cluster, the processing of this cluster gets a +// "free ride" because the thread responsible for processing +// the cluster that holds the object's header does the +// processing. +// d) in the case that the end of this cluster is spanned by a +// very large object, the processing of this cluster will +// be responsible for examining the entire object, +// potentially requiring this thread to process large amounts +// of memory pertaining to other clusters. +// +// Though an initial division of labor between marking threads may +// assign equal numbers of clusters to be scanned by each thread, it +// should be expected that some threads will finish their assigned +// work before others. Therefore, some amount of the full remembered +// set scanning effort should be held back and assigned incrementally +// to the threads that end up with excess capacity. Consider the +// following strategy for dividing labor: +// +// 1. Assume there are 8 marking threads and 1024 remembered +// set clusters to be scanned. +// 2. Assign each thread to scan 64 clusters. This leaves +// 512 (1024 - (8*64)) clusters to still be scanned. +// 3. As the 8 server threads complete previous cluster +// scanning assignments, issue each of the next 8 scanning +// assignments as units of 32 additional cluster each. +// In the case that there is high variance in effort +// associated with previous cluster scanning assignments, +// multiples of these next assignments may be serviced by +// the server threads that were previously assigned lighter +// workloads. +// 4. Make subsequent scanning assignments as follows: +// a) 8 assignments of size 16 clusters +// b) 8 assignments of size 8 clusters +// c) 16 assignments of size 4 clusters +// +// When there is no more remembered set processing work to be +// assigned to a newly idled worker thread, that thread can move +// on to work on other tasks associated with root scanning until such +// time as all clusters have been examined. +// +// Once all clusters have been processed, the gang of GC worker +// threads collaborate to merge the overreach data. +// +// for each ShenandoahHeapRegion old_region in the whole heap +// determine the cluster number of the first cluster belonging +// to that region +// for each cluster contained within that region +// Assure that exactly one worker thread initializes each +// cluster of overreach memory by invoking: +// +// rs->merge_overreach(cluster_no, cluster_count) +// +// in separate threads. (Divide up the clusters so that +// different threads are responsible for merging different +// clusters. Merging cost is essentially identical for +// each cluster.) +// +// Though remembered set scanning is designed to run concurrently with +// mutator threads, the current implementation of remembered set +// scanning runs in parallel during a GC safepoint. Furthermore, the +// current implementation of remembered set scanning never clears a +// card once it has been marked. Since the current implementation +// never clears marked pages, the current implementation does not +// invoke initialize_overreach() or merge_overreach(). +// +// These limitations will be addressed in future enhancements to the +// existing implementation. + +class ReferenceProcessor; +class ShenandoahConcurrentMark; +class ShenandoahHeap; +class CardTable; + +class ShenandoahDirectCardMarkRememberedSet: public CHeapObj { + +private: + + // Use symbolic constants defined in cardTable.hpp + // CardTable::card_shift = 9; + // CardTable::card_size = 512; + // CardTable::card_size_in_words = 64; + + // CardTable::clean_card_val() + // CardTable::dirty_card_val() + + ShenandoahHeap *_heap; + CardTable *_card_table; + uint32_t _card_shift; + size_t _total_card_count; + uint32_t _cluster_count; + HeapWord *_whole_heap_base; // Points to first HeapWord of data contained within heap memory + HeapWord *_whole_heap_end; + uint8_t *_byte_map; // Points to first entry within the card table + uint8_t *_byte_map_base; // Points to byte_map minus the bias computed from address of heap memory + uint8_t *_overreach_map; // Points to first entry within the overreach card table + uint8_t *_overreach_map_base; // Points to overreach_map minus the bias computed from address of heap memory + +public: + // count is the number of cards represented by the card table. + ShenandoahDirectCardMarkRememberedSet(CardTable *card_table, size_t total_card_count); + ~ShenandoahDirectCardMarkRememberedSet(); + + // Card index is zero-based relative to _byte_map. + uint32_t card_index_for_addr(HeapWord *p); + HeapWord *addr_for_card_index(uint32_t card_index); + bool is_card_dirty(uint32_t card_index); + void mark_card_as_dirty(uint32_t card_index); + void mark_card_as_clean(uint32_t card_index); + void mark_overreach_card_as_dirty(uint32_t card_index); + bool is_card_dirty(HeapWord *p); + void mark_card_as_dirty(HeapWord *p); + void mark_card_as_clean(HeapWord *p); + void mark_overreach_card_as_dirty(void *p); + uint32_t cluster_count(); + + // Called by multiple GC threads at start of concurrent mark and evacuation phases. Each parallel GC thread typically + // initializes a different subranges of all overreach entries. + void initialize_overreach(uint32_t first_cluster, uint32_t count); + + // Called by GC thread at end of concurrent mark or evacuation phase. Each parallel GC thread typically merges different + // subranges of all overreach entries. + void merge_overreach(uint32_t first_cluster, uint32_t count); +}; + +// A ShenandoahCardCluster represents the minimal unit of work +// performed by independent parallel GC threads during scanning of +// remembered sets. +// +// The GC threads that perform card-table remembered set scanning may +// overwrite card-table entries to mark them as clean in the case that +// the associated memory no longer holds references to young-gen +// memory. Rather than access the card-table entries directly, all GC +// thread access to card-table information is made by way of the +// ShenandoahCardCluster data abstraction. This abstraction +// effectively manages access to multiple possible underlying +// remembered set implementations, including a traditional card-table +// approach and a SATB-based approach. +// +// The API services represent a compromise between efficiency and +// convenience. +// +// In the initial implementation, we assume that scanning of card +// table entries occurs only while the JVM is at a safe point. Thus, +// there is no synchronization required between GC threads that are +// scanning card-table entries and marking certain entries that were +// previously dirty as clean, and mutator threads which would possibly +// be marking certain card-table entries as dirty. +// +// There is however a need to implement concurrency control and memory +// coherency between multiple GC threads that scan the remembered set +// in parallel. The desire is to divide the complete scanning effort +// into multiple clusters of work that can be independently processed +// by individual threads without need for synchronizing efforts +// between the work performed by each task. The term "cluster" of +// work is similar to the term "stripe" as used in the implementation +// of Parallel GC. +// +// Complexity arises when an object to be scanned crosses the boundary +// between adjacent cluster regions. Here is the protocol that is +// followed: +// +// 1. We implement a supplemental data structure known as the overreach +// card table. The thread that is responsible for scanning each +// cluster of card-table entries is granted exclusive access to +// modify the associated card-table entries. In the case that a +// thread scans a very large object that reaches into one or more +// following clusters, that thread has exclusive access to the +// overreach card table for all of the entries belonging to the +// following clusters that are spanned by this large object. +// After all clusters have been scanned, the scanning threads +// briefly synchronize to merge the contents of the overreach +// entries with the traditional card table entries using logical- +// and operations. +// 2. Every object is scanned in its "entirety" by the thread that is +// responsible for the cluster that holds its starting address. +// Entirety is in quotes because there are various situations in +// which some portions of the object will not be scanned by this +// thread: +// a) If an object spans multiple card regions, all of which are +// contained within the same cluster, the scanning thread +// consults the existing card-table entries and does not scan +// portions of the object that are not currently dirty. +// b) For any cluster that is spanned in its entirety by a very +// large object, the GC thread that scans this object assumes +// full responsibility for maintenance of the associated +// card-table entries. +// c) If a cluster is partially spanned by an object originating +// in a preceding cluster, the portion of the object that +// partially spans the following cluster is scanned in its +// entirety (because the thread that is responsible for +// scanning the object cannot rely upon the card-table entries +// associated with the following cluster). Whenever references +// to young-gen memory are found within the scanned data, the +// associated overreach card table entries are marked as dirty +// by the scanning thread. +// 3. If a cluster is spanned in its entirety by an object that +// originates within a preceding cluster's memory, the thread +// assigned to examine this cluster does absolutely nothing. The +// thread assigned to scan the cluster that holds the object's +// starting address takes full responsibility for scanning the +// entire object and updating the associated card-table entries. +// 4. If a cluster is spanned partially by an object that originates +// within a preceding cluster's memory, the thread assigned to +// examine this cluster marks the card-table entry as clean for +// each card table that is fully spanned by this overreaching +// object. If a card-table entry's memory is partially spanned +// by the overreaching object, the thread sets the card-table +// entry to clean if it was previously dirty and if the portion +// of the card-table entry's memory that is not spanned by the +// overreaching object does not hold pointers to young-gen +// memory. +// 5. While examining a particular card belonging to a particular +// cluster, if an object reaches beyond the end of its card +// memory, the thread "scans" all portions of the object that +// correspond to DIRTY card entries within the current cluster and +// all portions of the object that reach into following clustesr. +// After this object is scanned, continue scanning with the memory +// that follows this object if this memory pertains to the same +// cluster. Otherwise, consider this cluster's memory to have +// been fully examined. +// +// Discussion: +// Though this design results from careful consideration of multiple +// design objectives, it is subject to various criticisms. Some +// discussion of the design choices is provided here: +// +// 1. Note that remembered sets are a heuristic technique to avoid +// the need to scan all of old-gen memory with each young-gen +// collection. If we sometimes scan a bit more memory than is +// absolutely necessary, that should be considered a reasonable +// compromise. This compromise is already present in the sizing +// of card table memory areas. Note that a single dirty pointer +// within a 512-byte card region forces the "unnecessary" scanning +// of 63 = ((512 - 8 = 504) / 8) pointers. +// 2. One undesirable aspect of this design is that we sometimes have +// to scan large amounts of memory belonging to very large +// objects, even for parts of the very large object that do not +// correspond to dirty card table entries. Note that this design +// limits the amount of non-dirty scanning that might have to +// be performed for these very large objects. In particular, only +// the last part of the very large object that extends into but +// does not completely span a particular cluster is unnecessarily +// scanned. Thus, for each very large object, the maximum +// over-scan is the size of memory spanned by a single cluster. +// 3. The representation of pointer location descriptive information +// within Klass representations is not designed for efficient +// "random access". An alternative approach to this design would +// be to scan very large objects multiple times, once for each +// cluster that is spanned by the object's range. This reduces +// unnecessary overscan, but it introduces different sorts of +// overhead effort: +// i) For each spanned cluster, we have to look up the start of +// the crossing object. +// ii) Each time we scan the very large object, we have to +// sequentially walk through its pointer location +// descriptors, skipping over all of the pointers that +// precede the start of the range of addresses that we +// consider relevant. + + +// Because old-gen heap memory is not necessarily contiguous, and +// because cards are not necessarily maintained for young-gen memory, +// consecutive card numbers do not necessarily correspond to consecutive +// address ranges. For the traditional direct-card-marking +// implementation of this interface, consecutive card numbers are +// likely to correspond to contiguous regions of memory, but this +// should not be assumed. Instead, rely only upon the following: +// +// 1. All card numbers for cards pertaining to the same +// ShenandoahHeapRegion are consecutively numbered. +// 2. In the case that neighboring ShenandoahHeapRegions both +// represent old-gen memory, the card regions that span the +// boundary between these neighboring heap regions will be +// consecutively numbered. +// 3. (A corollary) In the case that an old-gen object spans the +// boundary between two heap regions, the card regions that +// correspond to the span of this object will be consecutively +// numbered. + + +// ShenandoahCardCluster abstracts access to the remembered set +// and also keeps track of crossing map information to allow efficient +// resolution of object start addresses. +// +// ShenandoahCardCluster supports all of the services of +// RememberedSet, plus it supports register_object() and lookup_object(). +// +// There are two situations under which we need to know the location +// at which the object spanning the start of a particular card-table +// memory region begins: +// +// 1. When we begin to scan dirty card memory that is not the +// first card region within a cluster, and the object that +// crosses into this card memory was not previously scanned, +// we need to find where that object starts so we can scan it. +// (Asides: if the objects starts within a previous cluster, it +// has already been scanned. If the object starts within this +// cluster and it spans at least one card region that is dirty +// and precedes this card region within the cluster, then it has +// already been scanned.) +// 2. When we are otherwise done scanning a complete cluster, if the +// last object within the cluster reaches into the following +// cluster, we need to scan this object. Thus, we need to find +// its starting location. +// +// The RememberedSet template parameter is intended to represent either +// ShenandoahDirectCardMarkRememberedSet, or a to-be-implemented +// ShenandoahBufferWithSATBRememberedSet. +template +class ShenandoahCardCluster: public CHeapObj { + +private: + RememberedSet *_rs; + +public: + static const uint32_t CardsPerCluster = 64; + + ShenandoahCardCluster(RememberedSet *rs) { + _rs = rs; + } + + ~ShenandoahCardCluster() { + } + + // There is one entry within the object_starts array for each + // card entry. The interpretation of the data contained within each + // object_starts entry is described below: + // + // Bits 0x003f: Value ranges from 0-63, which is multiplied by 8 + // to obtain the offset at which the first object + // beginning within this card region begins. + // Bits 0x0fc0: Value ranges from 0-63, which is multiplied by 8 to + // obtain the offset at which the last object beginning + // within this card region begins. + // Bits 0x8000: This bit is on if an object starts within this card + // region. + // + // In the most recent implementation of ShenandoahScanRemembered::process_clusters(), + // there is no need for the get_crossing_object_start() method function, so there is no + // need to maintain the following information. The comment is left in place for now in + // case we find it necessary to add support for this service at a later time. + // + // Bits 0x7fff: If no object starts within this card region, the + // remaining bits of the object_starts array represent + // the absolute word offset within the enclosing + // cluster's memory of the starting address for the + // object that spans the start of this card region's + // memory. If the spanning object begins in memory + // that precedes this card region's cluster, the value + // stored in these bits is the special value 0x7fff. + // (Note that the maximum value required to represent a + // spanning object from within the current cluster is + // ((63 * 64) - 8), which equals 0x0fbf. + // + // In the absence of the need to support get_crossing_object_start(), + // here is discussion of performance: + // + // Suppose multiple garbage objects are coalesced during GC sweep + // into a single larger "free segment". As each two objects are + // coalesced together, the start information pertaining to the second + // object must be removed from the objects_starts array. If the + // second object had been been the first object within card memory, + // the new first object is the object that follows that object if + // that starts within the same card memory, or NoObject if the + // following object starts within the following cluster. If the + // second object had been the last object in the card memory, + // replace this entry with the newly coalesced object if it starts + // within the same card memory, or with NoObject if it starts in a + // preceding card's memory. + // + // Suppose a large free segment is divided into a smaller free + // segment and a new object. The second part of the newly divided + // memory must be registered as a new object, overwriting at most + // one first_start and one last_start entry. Note that one of the + // newly divided two objects might be a new GCLAB. + // + // Suppose postprocessing of a GCLAB finds that the original GCLAB + // has been divided into N objects. Each of the N newly allocated + // objects will be registered, overwriting at most one first_start + // and one last_start entries. + // + // No object registration operations are linear in the length of + // the registered objects. + // + // Consider further the following observations regarding object + // registration costs: + // + // 1. The cost is paid once for each old-gen object (Except when + // an object is demoted and repromoted, in which case we would + // pay the cost again). + // 2. The cost can be deferred so that there is no urgency during + // mutator copy-on-first-access promotion. Background GC + // threads will update the object_starts array by post- + // processing the contents of retired GCLAB buffers. + // 3. The bet is that these costs are paid relatively rarely + // because: + // a) Most objects die young and objects that die in young-gen + // memory never need to be registered with the object_starts + // array. + // b) Most objects that are promoted into old-gen memory live + // there without further relocation for a relatively long + // time, so we get a lot of benefit from each investment + // in registering an object. + +private: + const uint32_t CardByteOffsetMultiplier = 8; + const uint32_t CardWordOffsetMultiplier = 1; + +#ifdef IMPLEMENT_THIS_OPTIMIZATION_LATER + + // This bit is set iff at least one object starts within a + // particular card region. + const uint_16 ObjectStartsInCardRegion = 0x8000; + const uint_16 FirstStartBits = 0x003f; + const uint_16 LastStartBits = 0x0fc0; + const uint_16 FirstStartShift = 0; + const uint_16 LastStartShift = 6; + const uint_16 CrossingObjectOverflow = 0x7fff; + + uint_16 *object_starts; + +public: + inline void set_first_start(uint32_t card_index, uint8_t value) { + object_starts[card_index] &= ~FirstStartBits; + object_starts[card_index] |= (FirstStartBits & (value << FirstStartShift)); + } + + inline void set_last_start(uint32_t card_index, uint8_t value) { + object_starts[card_index] &= ~LastStartBits; + object_starts[card_index] |= (LastStartBits & (value << LastStartShift)); + } + + inline void set_has_object_bit(uint32_t card_index) { + object_starts[card_index] |= ObjectStartsInCardRegion; + } + + // This has side effect of clearing ObjectStartsInCardRegion bit. + inline void set_crossing_object_start(uint32_t card_index, uint_16 crossing_offset) { + object_starts[card_index] = crossing_offset; + } +#endif // IMPLEMENT_THIS_OPTIMIZATION_LATER + +public: + + // The starting locations of objects contained within old-gen memory + // are registered as part of the remembered set implementation. This + // information is required when scanning dirty card regions that are + // spanned by objects beginning within preceding card regions. It + // is necessary to find the first and last objects that begin within + // this card region. Starting addresses of objects are required to + // find the object headers, and object headers provide information + // about which fields within the object hold addresses. + // + // The old-gen memory allocator invokes register_object() for any + // object that is allocated within old-gen memory. This identifies + // the starting addresses of objects that span boundaries between + // card regions. + // + // It is not necessary to invoke register_object at the very instant + // an object is allocated. It is only necessary to invoke it + // prior to the next start of a garbage collection concurrent mark + // or concurrent evacuation phase. An "ideal" time to register + // objects is during post-processing of a GCLAB after the GCLAB is + // retired due to depletion of its memory. + // + // register_object() does not perform synchronization. In the case + // that multiple threads are registering objects whose starting + // addresses are within the same cluster, races between these + // threads may result in corruption of the object-start data + // structures. Parallel GC threads should avoid registering objects + // residing within the same cluster by adhering to the following + // coordination protocols: + // + // 1. Align thread-local GCLAB buffers with some TBD multiple of + // card clusters. The card cluster size is 32 KB. If the + // desired GCLAB size is 128 KB, align the buffer on a multiple + // of 4 card clusters. + // 2. Post-process the contents of GCLAB buffers to register the + // objects allocated therein. Allow one GC thread at a + // time to do the post-processing of each GCLAB. + // 3. Since only one GC thread at a time is registering objects + // belonging to a particular allocation buffer, no locking + // is performed when registering these objects. + // 4. Any remnant of unallocated memory within an expended GC + // allocation buffer is not returned to the old-gen allocation + // pool until after the GC allocation buffer has been post + // processed. Before any remnant memory is returned to the + // old-gen allocation pool, the GC thread that scanned this GC + // allocation buffer performs a write-commit memory barrier. + // 5. Background GC threads that perform tenuring of young-gen + // objects without a GCLAB use a CAS lock before registering + // each tenured object. The CAS lock assures both mutual + // exclusion and memory coherency/visibility. Note that an + // object tenured by a background GC thread will not overlap + // with any of the clusters that are receiving tenured objects + // by way of GCLAB buffers. Multiple independent GC threads may + // attempt to tenure objects into a shared cluster. This is why + // sychronization may be necessary. Consider the following + // scenarios: + // + // a) If two objects are tenured into the same card region, each + // registration may attempt to modify the first-start or + // last-start information associated with that card region. + // Furthermore, because the representations of first-start + // and last-start information within the object_starts array + // entry uses different bits of a shared uint_16 to represent + // each, it is necessary to lock the entire card entry + // before modifying either the first-start or last-start + // information within the entry. + // b) Suppose GC thread X promotes a tenured object into + // card region A and this tenured object spans into + // neighboring card region B. Suppose GC thread Y (not equal + // to X) promotes a tenured object into cluster B. GC thread X + // will update the object_starts information for card A. No + // synchronization is required. + // c) In summary, when background GC threads register objects + // newly tenured into old-gen memory, they must acquire a + // mutual exclusion lock on the card that holds the starting + // address of the newly tenured object. This can be achieved + // by using a CAS instruction to assure that the previous + // values of first-offset and last-offset have not been + // changed since the same thread inquired as to their most + // current values. + // + // One way to minimize the need for synchronization between + // background tenuring GC threads is for each tenuring GC thread + // to promote young-gen objects into distinct dedicated cluster + // ranges. + // 6. The object_starts information is only required during the + // starting of concurrent marking and concurrent evacuation + // phases of GC. Before we start either of these GC phases, the + // JVM enters a safe point and all GC threads perform + // commit-write barriers to assure that access to the + // object_starts information is coherent. + + static void register_object(void *address, uint32_t length_in_bytes) { + +#ifdef IMPLEMENT_THIS_OPTIMIZATION_LATER + uint32_t card_at_start = cardAtAddress(address); + // The end of this object is contained within this card + uint32_t card_at_end = cardAtAddress(address + length_in_bytes); + + uint32_t cluster_at_start = clusterAtCard(card_at_start); + uint32_t cluster_at_end = clusterAtCard(card_at_end); + + void *card_start_address = addressAtCard(card_at_start); + void *cluster_start_address = addressAtCluster(cluster_at_start); + + uint8_t offset_in_card = + (address - card_start_address) / CardOffsetMultiplier; + + if (!getHashObjectBit(card_at_start)) { + set_has_object_bit(card_at_start); + set_first_start(card_at_start, offset_in_card); + set_last_start(card_at_start, offset_in_card); + } else { + if (offset_in_card < get_first_start(card_at_start)) + set_first_start(card_at_start, offset_in_card); + if (offset_in_card > get_last_start(card_at_start)) + set_last_start(card_at_last, offset_in_card); + } + +#ifdef SUPPORT_FOR_GET_CROSSING_OBJECT_START_NO_LONGER_REQUIRED + + // What is the last card within the current cluster? + uint32_t next_cluster = cluster_at_start + 1; + uint32_t card_at_next_cluster = cardAtCluster(next_cluster); + uint32_t last_card_of_this_cluster = card_at_next_cluster - 1; + uint32_t last_card_in_cluster = ((card_at_end < last_card_of_this_cluster) ? card_at_end: last_card_of_this_cluster); + + uint_16 crossing_map_within_cluster = address - cluster_at_start; + + for (uint32_t card_to_update = card_at_start + 1; card_to_update < last_card_in_cluster; card_to_update++) + set_crossing_object_start(card_to_update, crossing_map_within_cluster); + + // If the last card region of this cluster is completely spanned + // by this new object, set its crossing object start as well. + if (cluster_at_end > cluster_at_start) { + set_crossing_object_start(card_to_update++, crossing_map_within_cluster); + + // Now, we have to update all spanned cards that reside within the + // following clusters. + while (card_to_update < card_at_end) + set_crossing_object_start(card_to_update++, CrossingObjectOverflow); + + // Cases: + // + // 1. The region for card_at_end is not spanned at all because the + // end of the new object aligns with the start of the last + // card region. In this case, we do nothing with the + // card_at_end entry of the object_starts array. + // + // 2. The region for card_at_end is spanned partially. In this + // case, we do nothing with the card_at_end entry of the + // object_starts array. + // + // 3. Do not need to consider the case that the region for + // card_at_end is spanned entirely. If the last region + // spanned by a newly registered object is spanned in its + // entirety, then card_at_end will identify the card region + // that follows the object rather than the last region spanned + // by the object. + + // Bottom line: no further work to be done in any of these cases. + } + // Otherwise, the last card region of this cluster is not + // completely spanned by this new object. Leave the object_starts + // array entry alone. Two cases: + // + // a) This newly registered object represents a consolidation of + // multiple smaller objects, not including the object that + // follows the consolidation. + // b) This newly registered object is created by splitting a + // previously existing object. In this case, the other + // objects resulting from the split will be registered + // separately before it is necessary to lookup any information + // within the object_starts array. + // + +#endif // SUPPORT_FOR_GET_CROSSING_OBJECT_START_NO_LONGER_REQUIRED + +#else // IMPLEMENT_THIS_OPTIMIZATION_LATER + + // Do nothing for now as we have a brute-force implementation + // of findSpanningObject(). + +#endif // IMPLEMENT_THIS_OPTIMIZATION_LATER + } + + // The typical use case is going to look something like this: + // for each heapregion that comprises olg-gen memory + // for each card number that corresponds to this heap region + // scan the objects contained therein if the card is dirty + // To avoid excessive lookups in a sparse array, the API queries + // the card number pertaining to a particular address and then uses the + // card noumber for subsequent information lookups and stores. + + // Returns true iff an object is known to start within the card memory + // associated with addr p. + // Returns true iff an object is known to start within the card memory + // associated with addr p. + bool has_object(uint32_t card_index); + + // If has_object(card_index), this returns the word offset within this card + // memory at which the first object begins. + uint32_t get_first_start(uint32_t card_index); + + // If has_object(card_index), this returns the word offset within this card + // memory at which the last object begins. + uint32_t get_last_start(uint32_t card_index); + + // If !has_object(card_index), this returns the offset within the + // enclosing cluster at which the object spanning the start of this + // card memory begins, or returns 0x7fff if the spanning object + // starts before the enclosing cluster. + uint32_t get_crossing_object_start(uint32_t card_index); + + // Card index is zero-based relative to first spanned card region. + uint32_t card_index_for_addr(HeapWord *p); + HeapWord *addr_for_card_index(uint32_t card_index); + bool is_card_dirty(uint32_t card_index); + void mark_card_as_dirty(uint32_t card_index); + void mark_card_as_clean(uint32_t card_index); + void mark_overreach_card_as_dirty(uint32_t card_index); + bool is_card_dirty(HeapWord *p); + void mark_card_as_dirty(HeapWord *p); + void mark_card_as_clean(HeapWord *p); + void mark_overreach_card_as_dirty(void *p); + uint32_t cluster_count(); + void initialize_overreach(uint32_t first_cluster, uint32_t count); + void merge_overreach(uint32_t first_cluster, uint32_t count); +}; + +// ShenandoahScanRemembered is a concrete class representing the +// ability to scan the old-gen remembered set for references to +// objects residing in young-gen memory. +// +// In an initial implementation, remembered set scanning happens +// during a HotSpot safepoint. This greatly simplifies the +// implementation and improves efficiency of remembered set scanning, +// but this design choice increases pause times experienced at the +// start of concurrent marking and concurrent evacuation. Pause times +// will be especially long if old-gen memory holds many pointers to +// young-gen memory. +// +// Scanning normally begins with an invocation of numRegions and ends +// after all clusters of all regions have been scanned. +// +// Throughout the scanning effort, the number of regions does not +// change. +// +// Even though the regions that comprise old-gen memory are not +// necessarily contiguous, the abstraction represented by this class +// identifies each of the old-gen regions with an integer value +// in the range from 0 to (numRegions() - 1) inclusive. +// + +template +class ShenandoahScanRemembered: public CHeapObj { + +private: + + ShenandoahCardCluster *_scc; + +public: + // How to instantiate this object? + // ShenandoahDirectCardMarkRememberedSet *rs = + // new ShenandoahDirectCardMarkRememberedSet(); + // scr = new + // ShenandoahScanRememberd(rs); + // + // or, after the planned implementation of + // ShenandoahBufferWithSATBRememberedSet has been completed: + // + // ShenandoahBufferWithSATBRememberedSet *rs = + // new ShenandoahBufferWithSATBRememberedSet(); + // scr = new + // ShenandoahScanRememberd(rs); + + + ShenandoahScanRemembered(RememberedSet *rs); + ~ShenandoahScanRemembered(); + + // process_clusters() scans a portion of the remembered set during a JVM + // safepoint as part of the root scanning activities that serve to + // initiate concurrent scanning and concurrent evacuation. Multiple + // threads may scan different portions of the remembered set by + // making parallel invocations of process_clusters() with each + // invocation scanning different clusters of the remembered set. + // + // An invocation of process_clusters() examines all of the + // intergenerational references spanned by count clusters starting + // with first_cluster. The oops argument is assumed to represent a + // thread-local OopClosure into which addresses of intergenerational + // pointer values will be accumulated for the purposes of root scanning. + // + // A side effect of executing process_clusters() is to update the card + // table entries, marking dirty cards as clean if they no longer + // hold references to young-gen memory. (THIS IS NOT YET IMPLEMENTED.) + // + // The implementation of process_clusters() is designed to efficiently + // minimize work in the large majority of cases for which the + // associated cluster has very few dirty card-table entries. + // + // At initialization of concurrent marking, invoke process_clusters with + // ClosureType equal to ShenandoahInitMarkRootsClosure. + // + // At initialization of concurrent evacuation, invoke process_clusters with + // ClosureType equal to ShenandoahEvacuateUpdateRootsClosure. + + // This is big enough it probably shouldn't be in-lined. On the other hand, there are only a few places this + // code is called from, so it might as well be in-lined. The "real" reason I'm inlining at the moment is because + // the template expansions were making it difficult for the link/loader to resolve references to the template- + // parameterized implementations of this service. + template + void process_clusters(uint worker_id, ShenandoahReferenceProcessor* rp, ShenandoahConcurrentMark* cm, uint32_t first_cluster, uint32_t count, + HeapWord *end_of_range, ClosureType *oops); + + uint32_t cluster_for_addr(HeapWord *addr); + + // To Do: + // Create subclasses of ShenandoahInitMarkRootsClosure and + // ShenandoahEvacuateUpdateRootsClosure and any other closures + // that need to participate in remembered set scanning. Within the + // subclasses, add a (probably templated) instance variable that + // refers to the associated ShenandoahCardCluster object. Use this + // ShenandoahCardCluster instance to "enhance" the do_oops + // processing so that we can: + // + // 1. Avoid processing references that correspond to clean card + // regions, and + // 2. Set card status to CLEAN when the associated card region no + // longer holds inter-generatioanal references. + // + // To enable efficient implementation of these behaviors, we + // probably also want to add a few fields into the + // ShenandoahCardCluster object that allow us to precompute and + // remember the addresses at which card status is going to change + // from dirty to clean and clean to dirty. The do_oops + // implementations will want to update this value each time they + // cross one of these boundaries. + +}; + +typedef ShenandoahScanRemembered RememberedScanner; + +#endif // SHARE_GC_SHENANDOAH_SHENANDOAHSCANREMEMBERED_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp new file mode 100644 index 0000000000000..35e25d12a4867 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp @@ -0,0 +1,399 @@ +/* + * Copyright (c) 2021, Amazon.com, Inc. or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHSCANREMEMBEREDINLINE_HPP +#define SHARE_GC_SHENANDOAH_SHENANDOAHSCANREMEMBEREDINLINE_HPP + +#include "memory/iterator.hpp" +#include "oops/oop.hpp" +#include "oops/objArrayOop.hpp" +#include "gc/shenandoah/shenandoahCardTable.hpp" +#include "gc/shenandoah/shenandoahHeap.hpp" +#include "gc/shenandoah/shenandoahHeapRegion.hpp" +#include "gc/shenandoah/shenandoahScanRemembered.hpp" + +inline uint32_t +ShenandoahDirectCardMarkRememberedSet::card_index_for_addr(HeapWord *p) { + return (uint32_t) _card_table->index_for(p); +} + +inline HeapWord * +ShenandoahDirectCardMarkRememberedSet::addr_for_card_index(uint32_t card_index) { + return _whole_heap_base + CardTable::card_size_in_words * card_index; +} + +inline bool +ShenandoahDirectCardMarkRememberedSet::is_card_dirty(uint32_t card_index) { + uint8_t *bp = &_byte_map[card_index]; + return (bp[0] == CardTable::dirty_card_val()); +} + +inline void +ShenandoahDirectCardMarkRememberedSet::mark_card_as_dirty(uint32_t card_index) { + uint8_t *bp = &_byte_map[card_index]; + bp[0] = CardTable::dirty_card_val(); +} + +inline void +ShenandoahDirectCardMarkRememberedSet::mark_card_as_clean(uint32_t card_index) { + uint8_t *bp = &_byte_map[card_index]; + bp[0] = CardTable::clean_card_val(); +} + +inline void +ShenandoahDirectCardMarkRememberedSet::mark_overreach_card_as_dirty( + uint32_t card_index) { + uint8_t *bp = &_overreach_map[card_index]; + bp[0] = CardTable::dirty_card_val(); +} + +inline bool +ShenandoahDirectCardMarkRememberedSet::is_card_dirty(HeapWord *p) { + uint8_t *bp = &_byte_map_base[uintptr_t(p) >> _card_shift]; + return (bp[0] == CardTable::dirty_card_val()); +} + +inline void +ShenandoahDirectCardMarkRememberedSet::mark_card_as_dirty(HeapWord *p) { + uint8_t *bp = &_byte_map_base[uintptr_t(p) >> _card_shift]; + bp[0] = CardTable::dirty_card_val(); +} + +inline void +ShenandoahDirectCardMarkRememberedSet::mark_card_as_clean(HeapWord *p) { + uint8_t *bp = &_byte_map_base[uintptr_t(p) >> _card_shift]; + bp[0] = CardTable::clean_card_val(); +} + +inline void +ShenandoahDirectCardMarkRememberedSet::mark_overreach_card_as_dirty(void *p) { + uint8_t *bp = &_overreach_map_base[uintptr_t(p) >> _card_shift]; + bp[0] = CardTable::dirty_card_val(); +} + +inline uint32_t +ShenandoahDirectCardMarkRememberedSet::cluster_count() { + return _cluster_count; +} + + +template +inline uint32_t +ShenandoahCardCluster::card_index_for_addr(HeapWord *p) { + return _rs->card_index_for_addr(p); +} + +template +inline bool +ShenandoahCardCluster::has_object(uint32_t card_index) { + + HeapWord *addr = _rs->addr_for_card_index(card_index); + + ShenandoahHeap *heap = ShenandoahHeap::heap(); + ShenandoahHeapRegion *region = heap->heap_region_containing(addr); + + // Apparently, region->block_start(addr) is not robust to inquiries beyond top() and it crashes. + if (region->top() <= addr) + return false; + + HeapWord *obj = region->block_start(addr); + + // addr is the first address of the card region. + // obj is the object that spans addr (or starts at addr). + assert(obj != NULL, "Object cannot be null"); + if (obj >= addr) + return true; + else { + HeapWord *end_addr = addr + CardTable::card_size_in_words; + + // end_addr needs to be adjusted downward if top address of the enclosing region is less than end_addr. this is intended + // to be slow and reliable alternative to the planned production quality replacement, so go ahead and spend some extra + // cycles here in order to make this code reliable. + if (region->top() < end_addr) { + end_addr = region->top(); + } + + obj += oop(obj)->size(); + if (obj < end_addr) + return true; + else + return false; + } +} + +template +inline uint32_t +ShenandoahCardCluster::get_first_start(uint32_t card_index) { + HeapWord *addr = _rs->addr_for_card_index(card_index); + ShenandoahHeap *heap = ShenandoahHeap::heap(); + ShenandoahHeapRegion *region = heap->heap_region_containing(addr); + + HeapWord *obj = region->block_start(addr); + + assert(obj != NULL, "Object cannot be null."); + if (obj >= addr) + return obj - addr; + else { + HeapWord *end_addr = addr + CardTable::card_size_in_words; + obj += oop(obj)->size(); + + // If obj > end_addr, offset will reach beyond end of this card + // region. But clients should not invoke this service unless + // they first confirm that this card has an object. + assert(obj < end_addr, "Object out of range"); + return obj - addr; + } +} + +template +inline uint32_t +ShenandoahCardCluster::get_last_start(uint32_t card_index) { + HeapWord *addr = _rs->addr_for_card_index(card_index); + HeapWord *end_addr = addr + CardTable::card_size_in_words; + ShenandoahHeap *heap = ShenandoahHeap::heap(); + ShenandoahHeapRegion *region = heap->heap_region_containing(addr); + HeapWord *obj = region->block_start(addr); + assert(obj != NULL, "Object cannot be null."); + + if (region->top() <= end_addr) { + end_addr = region->top(); + } + + HeapWord *end_obj = obj + oop(obj)->size(); + while (end_obj < end_addr) { + obj = end_obj; + end_obj = obj + oop(obj)->size(); + } + assert(obj >= addr, "Object out of range."); + return obj - addr; +} + +template +inline uint32_t +ShenandoahCardCluster::get_crossing_object_start(uint32_t card_index) { + HeapWord *addr = _rs->addr_for_card_index(card_index); + uint32_t cluster_no = card_index / ShenandoahCardCluster::CardsPerCluster; + HeapWord *cluster_addr = _rs->addr_for_card_index(cluster_no * CardsPerCluster); + + ShenandoahHeap *heap = ShenandoahHeap::heap(); + ShenandoahHeapRegion *region = heap->heap_region_containing(addr); + HeapWord *obj = region->block_start(addr); + + if (obj > cluster_addr) + return obj - cluster_addr; + else + return 0x7fff; +} + +template +inline HeapWord * +ShenandoahCardCluster::addr_for_card_index(uint32_t card_index) { + return _rs->addr_for_card_index(card_index); +} + +template +inline bool +ShenandoahCardCluster::is_card_dirty(uint32_t card_index) { + return _rs->is_card_dirty(card_index); +} + +template +inline void +ShenandoahCardCluster::mark_card_as_dirty(uint32_t card_index) { + return _rs->mark_card_as_dirty(card_index); +} + +template +inline void +ShenandoahCardCluster::mark_card_as_clean(uint32_t card_index) { + return _rs->mark_card_as_clean(card_index); +} + +template +inline void +ShenandoahCardCluster::mark_overreach_card_as_dirty(uint32_t card_index) { + return _rs->mark_overreach_card_as_dirty(card_index); +} + +template +inline bool +ShenandoahCardCluster::is_card_dirty(HeapWord *p) { + return _rs->is_card_dirty(p); +} + +template +inline void +ShenandoahCardCluster::mark_card_as_dirty(HeapWord *p) { + return _rs->mark_card_as_dirty(p); +} + +template +inline void +ShenandoahCardCluster::mark_card_as_clean(HeapWord *p) { + return _rs->mark_card_as_clean(p); +} + +template +inline void +ShenandoahCardCluster::mark_overreach_card_as_dirty(void *p) { + return _rs->mark_overreach_card_as_dirty(p); +} + +template +inline uint32_t +ShenandoahCardCluster::cluster_count() { + return _rs->cluster_count(); +} + +template +inline void +ShenandoahCardCluster::initialize_overreach(uint32_t first_cluster, uint32_t count) { + return _rs->initialize_overreach(first_cluster, count); +} + +template +inline void +ShenandoahCardCluster::merge_overreach(uint32_t first_cluster, uint32_t count) { + return _rs->merge_overreach(first_cluster, count); +} + +template +ShenandoahScanRemembered::ShenandoahScanRemembered(RememberedSet *rs) { + _scc = new ShenandoahCardCluster(rs); +} + +template +ShenandoahScanRemembered::~ShenandoahScanRemembered() { + delete _scc; +} + +template +template +inline void +ShenandoahScanRemembered::process_clusters(uint worker_id, ShenandoahReferenceProcessor* rp, ShenandoahConcurrentMark* cm, + uint32_t first_cluster, uint32_t count, HeapWord *end_of_range, + ClosureType *oops) { + + // Unlike traditional Shenandoah marking, the old-gen resident objects that are examined as part of the remembered set are not + // themselves marked. Each such object will be scanned only once. Any young-gen objects referenced from the remembered set will + // be marked and then subsequently scanned. + + while (count-- > 0) { + uint32_t card_index = first_cluster * ShenandoahCardCluster::CardsPerCluster; + uint32_t end_card_index = card_index + ShenandoahCardCluster::CardsPerCluster; + + first_cluster++; + int next_card_index = 0; + while (card_index < end_card_index) { + int is_dirty = _scc->is_card_dirty(card_index); + int has_object = _scc->has_object(card_index); + + if (is_dirty) { + if (has_object) { + // Scan all objects that start within this card region. + uint32_t start_offset = _scc->get_first_start(card_index); + HeapWord *p = _scc->addr_for_card_index(card_index); + HeapWord *endp = p + CardTable::card_size_in_words; + if (endp > end_of_range) { + endp = end_of_range; + next_card_index = end_card_index; + } else { + // endp either points to start of next card region, or to the next object that needs to be scanned, which may + // reside in some successor card region. + next_card_index = _scc->card_index_for_addr(endp); + } + + p += start_offset; + while (p < endp) { + oop obj = oop(p); + // Future TODO: + // For improved efficiency, we might want to give special handling of obj->is_objArray(). In + // particular, in that case, we might want to divide the effort for scanning of a very long object array + // between multiple threads. + if (obj->is_objArray()) { + ShenandoahObjToScanQueue* q = cm->get_queue(worker_id); + ShenandoahMarkRefsClosure cl(q, rp); + objArrayOop array = objArrayOop(obj); + int len = array->length(); + array->oop_iterate_range(&cl, 0, len); + } else { + oops->do_oop(&obj); + } + p += obj->size(); + } + card_index = next_card_index; + } else { + // otherwise, this card will have been scanned during scan of a previous cluster. + card_index++; + } + } else if (_scc->has_object(card_index)) { + // Scan the last object that starts within this card memory if it spans at least one dirty card within this cluster + // or if it reaches into the next cluster. + uint32_t start_offset = _scc->get_last_start(card_index); + HeapWord *p = _scc->addr_for_card_index(card_index) + start_offset; + oop obj = oop(p); + HeapWord *nextp = p + obj->size(); + uint32_t last_card = _scc->card_index_for_addr(nextp); + + bool reaches_next_cluster = (last_card > end_card_index); + bool spans_dirty_within_this_cluster = false; + + if (!reaches_next_cluster) { + uint32_t span_card; + for (span_card = card_index+1; span_card < end_card_index; span_card++) + if (_scc->is_card_dirty(span_card)) { + spans_dirty_within_this_cluster = true; + break; + } + } + if (reaches_next_cluster || spans_dirty_within_this_cluster) { + if (obj->is_objArray()) { + ShenandoahObjToScanQueue* q = cm->get_queue(worker_id); // kelvin to confirm: get_queue wants worker_id + ShenandoahMarkRefsClosure cl(q, rp); + objArrayOop array = objArrayOop(obj); + int len = array->length(); + array->oop_iterate_range(&cl, 0, len); + } else { + oops->do_oop(&obj); + } + } + // Increment card_index to account for the spanning object, even if we didn't scan it. + card_index = (last_card > card_index)? last_card: card_index + 1; + } else { + card_index++; + } + } + } +} + +template +inline uint32_t +ShenandoahScanRemembered::cluster_for_addr(HeapWordImpl **addr) { + uint32_t card_index = _scc->card_index_for_addr(addr); + uint32_t result = card_index / ShenandoahCardCluster::CardsPerCluster; + return result; +} + +#endif // SHARE_GC_SHENANDOAH_SHENANDOAHSCANREMEMBEREDINLINE_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahStringDedup.cpp b/src/hotspot/share/gc/shenandoah/shenandoahStringDedup.cpp index c715e7378b163..d2c373b16bf52 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahStringDedup.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahStringDedup.cpp @@ -47,21 +47,10 @@ void ShenandoahStringDedup::enqueue_candidate(oop java_string) { assert(Thread::current()->is_Worker_thread(), "Only from a GC worker thread"); - if (java_string->age() <= StringDeduplicationAgeThreshold) { - const markWord mark = java_string->mark(); - - // Having/had displaced header, too risk to deal with them, skip - if (mark == markWord::INFLATING() || mark.has_displaced_mark_helper()) { - return; - } - - // Increase string age and enqueue it when it rearches age threshold - markWord new_mark = mark.incr_age(); - if (mark == java_string->cas_set_mark(new_mark, mark)) { - if (mark.age() == StringDeduplicationAgeThreshold) { - StringDedupQueue::push(ShenandoahWorkerSession::worker_id(), java_string); - } - } + // String age is incremented when object is evacuated as with all objects. + if (java_string->age() >= StringDeduplicationAgeThreshold) { + // Enqueue string for deduplication when age reaches age threshold. + StringDedupQueue::push(ShenandoahWorkerSession::worker_id(), java_string); } } From 28fc3420f0c393e3d46c75a4c9a67317cc5f5824 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Thu, 4 Feb 2021 08:44:41 +0000 Subject: [PATCH 009/254] Milestone 1 Reviewed-by: rkennke, adityam --- genshen-docs/README.md | 9 + genshen-docs/design.summary.md | 329 +++++++++++++++ genshen-docs/glossary.md | 44 ++ genshen-docs/workplan.summary.md | 395 ++++++++++++++++++ .../share/gc/shared/gcConfiguration.cpp | 9 +- .../share/gc/shenandoah/shenandoahHeap.cpp | 3 +- test/hotspot/jtreg/TEST.groups | 16 + .../generational/TestCLIModeGenerational.java | 53 +++ .../generational/TestSimpleGenerational.java | 129 ++++++ 9 files changed, 985 insertions(+), 2 deletions(-) create mode 100644 genshen-docs/README.md create mode 100644 genshen-docs/design.summary.md create mode 100644 genshen-docs/glossary.md create mode 100644 genshen-docs/workplan.summary.md create mode 100644 test/hotspot/jtreg/gc/shenandoah/generational/TestCLIModeGenerational.java create mode 100644 test/hotspot/jtreg/gc/shenandoah/generational/TestSimpleGenerational.java diff --git a/genshen-docs/README.md b/genshen-docs/README.md new file mode 100644 index 0000000000000..ec077704b35ce --- /dev/null +++ b/genshen-docs/README.md @@ -0,0 +1,9 @@ +## GenShen: the Generational Shenandoah project + +Gen Shen stands for Generational Shenandoah, +and it also means "deep roots" in Mandarin. + +Documents in this directory: +- [glossary.md](glossary.md): glossary of terms used in our documents, +- [workplan.summary.md](workplan.summary.md): summary of the plan of record that we strive to follow, +- [design.summary.md](design.summary.md): summary of design points that we rely upon as working hypotheses. diff --git a/genshen-docs/design.summary.md b/genshen-docs/design.summary.md new file mode 100644 index 0000000000000..22393046c0a42 --- /dev/null +++ b/genshen-docs/design.summary.md @@ -0,0 +1,329 @@ +## Design Points + +This section discusses design decisions, as guided by the following tenets. + +### Tenets + +1. Don’t punish the mutator: While mutator code must be added to + implement generational collection, degradations of + mutator throughput and responsiveness + should be contained as much as possible. + If in doubt between design choices, + pick the one that promises to minimize mutator overhead. +2. Safeguard incremental progress with milestones that demonstrate key capabilities. +3. Minimize the overall development effort. This includes containing + efforts to prepare and present each milestone deliverable + that do not directly contribute to the end product. + +### Design Decisions + +While it is probable that we may revisit some of the following design +decisions if we run into implementation difficulties or performance +problems, the following is the current plan of record. + +1. This document is maintained alongside the source code. + 1. It is organized hierarchically with sections + dedicated to major design decisions. + Further, more detailed decisions are organized in top-down fashion, + with overarching design decisions preceding derivative design decisions. + The topic numbers assigned to particular design decisions may evolve over time. + 2. There is a separate "rationale" document that explains more of the reasoning behind design decisions + and links relevant portions between documents. +2. Genshen changes to the Shenandoah implementation will be + compartmented. See discussion [here](rationale.summary.md#compartmentalization"). +3. We will support concurrent collection of old gen and young gen, + with the expectation that a single old collection will typically + overlap with the execution of many young collections. +4. In order to minimize performance impact on the mutator, we will use + a single Load-Reference-Barrier to implement both evacuation + from young gen and from old gen. + 1. Tenuring will be implemented as part of young evacuation. + 2. All evacuation for both young and old gen regions + happens under the control of the same GC threads and is + supported by the same Load Reference Barrier (LRB) as in + vanilla Shenandoah, with only small refinements to the + implementation of slow-path code within the LRB. + 3. See discussion [here](rationale.summary.md#load-reference-barrier). +5. An old collection begins at the same time as a young gen + collection, with both collections leveraging the + results of a single shared root scan. +6. Old collection generally runs at lower priority (e.g., fewer + numbers of parallel threads or greater “nice” values for + concurrent threads) than young collection because + replenishing the allocation pool to support ongoing allocation + needs of mutator threads requires urgent completion of young GC. + See discussion [here](rationale.summary.md#prioritization). +7. Although the planned future production release of GenShen will + run young collection concurrently with old collection, support + will also be implemented and maintained for alternating executions + of young collections with global collections. See + discussion [here](rationale.summary.md#young-global-discussion). + 1. Since this is not a production release, efficiency of + implementation is not a primary concern. + 2. Regression tests will exercise the implementation of this mode + of operation. +8. A regression test suite will be developed and maintained to + support all of the enhanced capabilities of GenShen, including + capabilities that are enabled and disabled at build time for the + JVM. +9. Though old collections start at the same time as young + collections collections, they do not necessarily end at the same + time. Typically, many young-gen collections will be completed + concurrently during the time that a single old-gen collection is + running. See discussion [here](rationale.summary.md#concurrency-of-young-and-old). +10. Root scanning will be performed during a JVM safe point. +11. Scanning of the remembered set will be performed by parallel + young-gen GC threads during a JVM safe point. A side + effect of scanning is to mark as CLEAN any cards of the remembered + set that are found by remembered set scanning to no longer be + DIRTY. +12. The remembered set maintains a start-of-object representation to + facilitate quick identification of where the first object + pertaining to each DIRTY card region begins. See discussion + [here](rationale.summary.md#remembered-set-starting-offsets). + This requires that: + 1. Newly promoted objects be registered with the remembered set + and start-of-object support by post-processing the associated GCLAB block, and + 2. When entire regions are promoted out of young collections into + old gen, the objects contained within those regions must be + registered with the remembered set and the start-of-object support. +13. Young marking, including scanning of root pointers, + will place discovered references to old gen into a thread-local SATB + buffer so that they can be processed by an old-gen collection + thread. See discussion [here](rationale.summary.md#satb-keep-old-alive-entries). + 1. Possible optimization: refrain from logging old-gen pointers + that refer to already marked old objects into the SATB buffer. +14. The default size of the SATB buffer will increase from 1024 to + 4096 words because we will be placing more information into the + SATB buffer. +15. Whenever the SATB buffer is full, the slow path for adding to + the SATB buffer will attempt to compress the buffer contents before + communicating the buffer to GC threads. If compression leaves a + minimum of 256 slots available within the SATB buffer, the thread + continues to add values to the existing buffer. Compression + consists of: + 1. For pointers to young gen: + 1. If concurrent marking of young gen is disabled, ignore the + pointer. + 2. Otherwise, if the object referenced by this pointer has + already been marked, ignore the pointer. + 3. If we choose to use SATB-based remembered set, ignore all + overwritten address values that reside within young gen. + 2. For pointers to old gen: + 1. If concurrent marking of old gen is disabled, ignore the + pointer. + 2. Otherwise, if the object referenced by this pointer + has already been marked, ignore the pointer. + 3. If we choose to use an SATB-based remembered set: + - If the card corresponding to an overwritten old gen + address is already DIRTY, ignore this overwritten + address. + - Otherwise, fetch the pointer value held at the overwritten + address. If the fetched pointer value does not + refer to young gen, ignore this overwritten address. + - Otherwise, use a hash table (suggested size 127) to sift + redundant DIRTY card references for the current batch of + overwritten old-gen addresses. Preserve only one address + for each card entry that needs to be marked as DIRTY. + 3. SATB buffers will be processed by both a young GC + thread and an old GC thread. + 1. The young GC thread will mark objects referenced + by young pointers. + 2. The old GC thread will: + 1. Mark objects referenced by old-gen pointers, and + 2. If we choose to use SATB-based remembered set: mark as DIRTY + the card entries corresponding to overwritten addresses. +16. GC uses a G1-style heuristic to choose collection sets for both + young and old memory areas. The collection set represents the + heap regions that offer the greatest opportunity to quickly reclaim + garbage, as with the existing Shenandoah implementation. See + discussion [here](rationale.summary.md#g1-heuristic). +17. Following an evacuation phase that evacuates + both old and young heap regions, the update-references phase is + required to update references throughout all + old regions that were not selected as part of the old + collection set in addition to updating references in young + heap regions. + 1. If a particular young collection evacuation phase does not + evacuate any old regions, then its update references phase can + focus solely on young heap regions and the + remembered set. +18. Tenuring is based on object age with the enhancements described + below. See discussion [here](rationale.summary.md#tenuring). + 1. The variable TenureCycle counts how many GC cycles correspond to + each increment of an object’s age. Object ages are not + necessarily incremented each time a GC cycle is completed. They + are incremented each time TenureCycle GC cycles have been + completed. + 2. The variable TenureAge represents the age at which an object + becomes eligible for tenuring. + 3. During GC cycles that correspond to TenureCycle, the “age” of + individual objects is incremented by 1 plus the size of the + object’s original heap region age when the object is evacuated. + 4. During GC cycles that correspond to TenureCycle, the “age” of + each heap region that has been fully allocated (i.e. no more + available memory for allocation) and that is not in the + collection set is incremented by 1 at the end of the evacuation + phase. If the resulting “age” equals TenureAge, then the entire + region is reassigned to become part of old gen. + 1. The update-refs phase will process this heap region even + though it is “now” considered to be part of old gen. + 2. Each of the objects contained within the promoted region + shall be registered with the remembered set abstraction. + 3. Each of the objects contained within the promoted region + be scanned to determine any references to young-gen + memory that are contained therein. For any such pointers, set + the associated card table entry to DIRTY. +19. During evacuation, each running mutator thread has both a TLAB + and a GCLAB. + 1. The TLAB is associated with a young heap region. + 2. The GCLAB is associated with an old heap region. + 3. When the mutator’s load reference barrier encounters a + reference for which the associated object needs to be tenured, it + allocates the copy memory from the GCLAB. + 4. When the mutator’s load reference barrier encounters a + reference for which the associated object resides within the + collection set of old gen, it allocates the copy memory from the + GCLAB. + 5. When the mutator’s load reference barrier encounters a + reference for which the associated object needs to be evacuated to + young gen, it allocates the copy memory from the TLAB. + 6. We initially plan to use the same size for TLAB and GCLAB, but + this decision may be revisited. + 7. If the size of the object to be evacuated is larger than half + the size of the respective local allocation buffer, allocation + of the replica memory is handled by alternative allocators, to be + designed. Call these "odd" objects. +20. During evacuation, each evacuating GC thread will maintain two + GCLAB buffers: + 1. GCLAB-Young is associated with young gen. + 2. GCLAB-Old is associated with old gen. + 3. If the object to be evacuated currently resides in old gen or + if it resides in young gen and it is to be tenured, allocate the + copy memory from GCLAB-Old. + 4. Otherwise, allocate the copy memory from GCLAB-Young. + 5. At the end of the evacuation phase, consider repurposing any + unspent GCLAB-Young as a TLAB if there is sufficient unallocated + memory remaining within it. + 6. At the end of the evacuation phase, consider preserving the + GCLAB-Old for use as a GCLAB for a mutator thread during the next + young collections collection or as a GCLAB-Old during the next + old-gen evacuation pass. + 7. See 19.7. +21. A certain budget of CPU time is provisioned to perform young-gen + GC in order to support a particular planned workload. + 1. The resources dedicated to young-gen GC are limited in order + to assure a certain amount of CPU time is available to mutator + threads. + 2. Command line options allow user control over provisioning. A + TBD API may allow services to adjust provisioning dynamically. + 3. In the ideal, provisioning is adjusted automatically based on + TBD heuristics. + 4. The provisioned CPU resources can support a range of service + quality, reclaiming large numbers of heap regions with a low GC + frequency or reclaiming small numbers of heap regions with a + high GC frequency. Given a particular frequency of GC cycles, + the same CPU resources can evacuate a large number of sparsely + populated heap regions or a small number of densely populated + heap regions. Tradeoffs between these configuration extremes + may be adjusted under software control or by TBD + heuristics. + 5. For each young-gen GC pass, a certain TBD percentage + of CPU resources are reserved for old-gen evacuation and + update-refs activities. + 1. The old-gen CPU resource budget represents the total amount + of old-gen memory that can be relocated, and is quantified as + a multiple N of the heap region size. + 2. The old-gen GC threads determine the composition of the + old-gen collection set, up to but never exceeding the upper + bound N on cumulative evacuation size. + 3. The old-gen GC thread may select for the collection set + N heap regions which are known to have 100% + utilization, 2N heap regions known to have 50% utilization, + 5N heap regions known to have 20% utilization, and so on. + 4. If the old-gen GC refrains from delegating N heap regions + worth of evacuation work to the young-gen evacuation phase, + then the young GC is free to use the excess CPU resources to + more aggressively evacuate more of its own young heap regions, + using a larger than normal young-gen collection set. + 5. The budget for updating old-gen references must be large + enough to handle the total old-gen memory size - N. In the + case that old-gen GC configures a collection set that + represents its full evacuation capacity of N heap regions, the + number of old-gen heap regions that are not part of the + old-gen collection set is never larger than this quantity. + In the case that old-gen GC configures a smaller collection + set, then for each fraction of a heap region that is not + evacuated, this much more of a heap region might have to be + processed during the update-refs phase of GC. We estimate + that the cost of evacuating M bytes of memory is similar to + the cost of updating the references within M bytes of + memory. +22. A smaller (generally) budget of CPU time is provisioned to + perform old-gen GC in order to support a particular planned + workload. + 1. The resources dedicated to young-gen GC are limited in order + to assure a certain amount of CPU time is available to mutator + threads. + 2. Command-line options allow user control over provisioning. A + TBD API may allow services to adjust provisioning dynamically. + 3. In the ideal, provisioning is adjusted automatically based on + TBD heuristics. + 4. As with young-gen GC, the CPU resources provisioned for + old-gen GC can support a range of service quality. + 5. The CPU resources dedicated to old-gen collection do not have + responsibility for evacuating old regions as all evacuation + is performed by young-gen GC threads. Instead, the CPU + resources dedicated to old-gen GC activities are used for + activities such as the following: + 1. Processing the content of SATB buffers: + - If old collection is in concurrent marking phase, mark + objects referenced by any keep-alive pointers. + - If we are using SATB-based remembered set, update the + remembered set based on overwritten addresses reported in the + SATB buffer. + 2. During concurrent marking, scan the contents of previously + marked objects to complete the transitive closure of + reachability. + 3. Perform heuristic computations: + - Determine when to start the next old-gen GC cycle. + - Determine which old regions to evacuate on this and future + passes of young-gen GC. + - Adjust young-gen efficiency parameters such as: How many + heap regions should be dedicated to young gen? What is + optimal value of TenureCycle? What is optimal value of + TenureAge? How much CPU time should be dedicated to + young-gen GC? + - Adjust old-gen efficiency parameters such as: How much CPU + time should be dedicated to old-gen GC? How many heap regions + should be dedicated to old gen? Should any heap regions be + decommissioned and returned to the operating system in order + to shrink the memory footprint of this service? + 4. Perform routine maintenance as time and schedule permits: + - Potentially sweep up dead memory, accumulating ages at + which dead objects were reclaimed within old regions. + - Potentially, sweep through evacuated memory to accumulate + ages at which dead objects were reclaimed. + - Organize free lists for fast and efficient allocation of + GCLAB and Odd object allocations. + - Return emptied old regions to the free set. + - Eventually, reference processing and string deduplication + should be performed by lower priority old-gen threads + rather than higher priority young-gen threads. + 5. Following completion of each old-gen concurrent mark pass, + select regions to be included in the old-gen collection set: + - No more than a total of N bytes of old gen is evacuated by + each pass of the young-gen evacuator. + - If old-gen GC threads desire to evacuate M, which is more + than N bytes of old gen, it does so by piggy backing on + multiple subsequent young-gen evacuation passes, selecting + evacuating no more than the accumulation of N total heap + regions in each of the following young-gen evacuation passes. + - Since it is most efficient to evacuate all M > N regions + of old-gen memory with a single young-gen evacuation pass, + configure the old-gen collection set to include all M + regions if there are sufficient free regions available to + afford the young-gen allocator to continue allocating new + objects during the longer delay that would be required to + evacuate more than the traditionally budgeted N regions of + old-gen memory. diff --git a/genshen-docs/glossary.md b/genshen-docs/glossary.md new file mode 100644 index 0000000000000..f8b98564557f0 --- /dev/null +++ b/genshen-docs/glossary.md @@ -0,0 +1,44 @@ +## Glossary for the Generational Shenandoah (GenShen) Project + +Shen := Shenandoah := the Shenandoah garbage collector + +GenShen := Generational Shenandoah + +gen := generation +young gen := the young generation +old gen := the old generation + +collector := garbage collector +young collector := young gen collector +old collector := old gen collector +global collector := single-generation collector that works on the entire heap + +young/old/global collection := a complete cycle through the phases of the young/old/global collector + +cset := collection set +remset := remembered set +rset := remembered set + +parallel := same task, dealt with by multiple threads +concurrent := different tasks, operated upon simultaneously +conc := concurrent + +conc mark := concurrent marking +conc global GC := concurrent global GC, like vanilla Shen +evac := evacuation (phase) +UR := update references (phase) + +LRB := Load Reference Barrier +SATB := Snapshot At The Beginning +TLAB := Thread-Local Allocation Buffer for a mutator thread +GCLAB := like a TLAB, but for a GC thread + +young region := a heap region affiliated with young gen +old region := a heap region affiliated with old gen +free region := a region that is not affiliated with either generation and available for future reuse by allocators + +young object := an object in young gen +old object := an object in old gen + +block := an identifiable chunk of space in a region that is or was occupied by a Java object +block start := a pointer to the beginning of a block diff --git a/genshen-docs/workplan.summary.md b/genshen-docs/workplan.summary.md new file mode 100644 index 0000000000000..dde3c0c0d3b3d --- /dev/null +++ b/genshen-docs/workplan.summary.md @@ -0,0 +1,395 @@ +# Summary Plan of Record: GenShen Prototype (2020) + +## Planned Milestones + +### Overview of Initial Milestones + +1. Pure young collection. (Young gen size unlimited. Old gen untouched.) +2. Size-restricted young gen. +3. Tenuring and promotion. +4. Global collection after young collection. +5. Young collection after global collection, repeat alternations. +6. Concurrent old marking. +7. Concurrent old and young collections. + +Young collection reclaims garbage only from heap regions that are +identified as belonging to the young generation. + +Old collection reclaims garbage only from heap regions that are +identified as belonging to the old generation, which holds objects +that have been promoted from young generation. + +Global collection reclaims garbage from heap regions belonging to +either the young generation or the old generation. + +### Milestone 1: GC of Young Gen Only + +This demonstration proves successful implementation of: + +1. Separating out that certain heap regions comprise young gen and + other heap regions are considered to represent old gen. +2. Confirming that card marking does not crash. (Does not prove that + card marking works because we have no objects in old gen.) +3. Confirming that remembered set scanning does not crash. (Does not + prove that remembered set scanning works because we have no objects in + old gen.) +4. Demonstrating a simplified form of young collection. + +Tasks + +1. Integrate various completed code patches into a common branch +2. Test and debug existing code + +### Milestone 2: Restrict Size of Young Gen + +This milestone constrains the young generation to a certain number of +regions. After that number is reached, allocation fails. For now, the +number can be fixed, say 25% of all regions, and we can simply crash +thereafter. + +Tasks + +1. Establish a mechanism to specify and enforce a limit on the number + of heap regions that may comprise young-gen memory. +2. Adjust the GC triggering mechanism so that the size of young gen + does not have reason to exceed the young-gen size during young-gen + GC, where the size of young-gen is affected by GC in the + following ways: + 1. Certain free heap regions may be added to young gen in order to + support allocation requests that are made by concurrent mutator + threads while GC is being performed. + 2. At the end of GC, all heap regions that were part of the + collection set are removed from young-gen memory and placed + in the free pool. + + +### Milestone 3: Young Collection with Promotion of Tenured Objects + +Add to Milestone 2 the capability of promoting young gen objects. +Don’t worry about odd objects or humongous objects at this time. +Demonstrate: + +1. That we can promote objects into old gen. +2. That card-marking works. +3. That remembered set scanning works (correctly, but perhaps not with + full desired efficiency). + +The following tasks must be implemented: + +1. The collection set is selected as a subset of all young collections + heap regions using TBD heuristics. +2. During young collection, each time an object is evacuated to a + young consolidation region, the object’s age is incremented. + 1. For simplicity, don’t try to implement the complete region + aging or promotion at this time. + 2. Also for simplicity, don’t try to implement the TenureCycle + optimization. +3. During young collection, each GC thread maintains both a young + GCLAB and and old GCLAB. + 1. When evacuating an object whose incremented age is less than + TenureAge, allocate memory for the object’s copy from within the + young GCLAB. + 2. When evacuating an object whose incremented age is >= + TenureAge, allocate memory for the object’s copy from within the + old GCLAB. + 3. Don’t support Odd or Humongous objects for simplicity. +4. During young collection, each mutator thread maintains both a TLAB + and a GCLAB. The GCLAB is associated with old gen. + 1. When evacuating an object whose incremented age is less than + TenureAge, allocate memory for the object’s copy from within the + TLAB. + 2. When evacuating an object whose incremented age is >= + TenureAge, allocate memory for the object’s copy from within the + GCLAB. +5. Perform maintenance on the contents of each retired old GCLAB, + where this maintenance consists of: + 1. Registering each object’s starting location and length with the + remembered set abstraction so that remembered set scanning can + quickly find the first object within each DIRTY card region, + 2. Updating the card table to reflect all references from this + object into young gen, + 3. Evacuating all objects directly referenced from this object + which reside within a collection set and have not yet been + evacuated, and + 4. Healing all pointers from within newly evacuated old objects + that refer to objects residing within the collection set. + 5. The last two items listed above are already performed by + traditional Shenandoah but can be merged with the implementation of + the other maintenance activities in order to perform all this work + in a consolidated pass. + +### Milestone 4: Sequential Execution of Multiple Young Collections Followed by Multiple Global Collections + +__Demonstrated Concurrency Between Young-Gen and Old-Gen Activities__ + + ✓ denotes that this combination of activities is allowed. + + ✗ denotes that this combination of activities is disallowed. + +| | Old-Gen Mark | Old-Gen Evac | Old-Gen Idle | +|:--------------:|:--------------:|:-------------:|:------------:| +| Young Gen Mark | ✓ | ✗ | ✓ | +| Young Gen Evac | ✗ | ✓ | ✓ | +| Young Gen Idle | ✗ | ✗ | ✓ | + +Add to Milestone 3 the ability to switch to global collection after a +series of young Collections. + +1. The switch is triggered by a simple TBD test, such as when the + space available within old gen is less than the size of young gen. +2. The global collection continues to run with the card-marking + barrier enabled though the card values will not be consulted further. +3. For this demonstration, the switch to global collections is + one-way. Following this switch, we can no longer perform young + collections. +4. For this demonstration, the global collection does not distinguish + between young regions and old regions. + 1. Evacuation of objects that resided in an old region is handled + the same as evacuation of objects that resided in a young heap + region. +5. This demonstrates that: + 1. Promotion of objects works. The objects that have been promoted + into old gen maintain whatever protocols and invariants are + assumed by Shenandoah GC. + 2. That we can manage the transition from young GC to global GC. + 3. That global collection, insofar as we characterize it, works. + 4. That Young collections do not corrupt the heap. + +Tasks: + +1. Fix bugs in existing code enhancements. +2. Implement the transition from young collection to global collection +3. Implement global collection with write barrier for card-table marking + +### Milestone 5: Interleaved Execution of Young and Global Collections + +__Demonstrated Concurrency Between Young-Gen and Old-Gen Activities__ + + ✓ denotes that this combination of activities is allowed. + + ✗ denotes that this combination of activities is disallowed. + +| | Old-Gen Mark | Old-Gen Evac | Old-Gen Idle | +|:--------------:|:--------------:|:-------------:|:------------:| +| Young Gen Mark | ✓ | ✗ | ✓ | +| Young Gen Evac | ✗ | ✓ | ✓ | +| Young Gen Idle | ✗ | ✗ | ✓ | + +Add to Milestone 4 the ability to switch back to Young Collection upon +completion of a global collection. + +1. Assume that the global collection is successful in reclaiming + necessary memory. No heuristics to resize oldgen or young gen at + this time. Specify sizes of each on command line. +2. The switch to old collection is triggered by exhaustion of old gen. + At least one young collection is assumed to execute following + completion of each global collection. +3. For this demonstration, the global collection does distinguish + between young collections and old regions. + 1. Objects in the old collection set are evacuated to old + consolidation regions. + 2. Objects in the young collection set are evacuated + to young collections consolidation regions. + 3. There is no tenuring of objects during a global collection (for + simplicity). + +This demonstrates that: + +1. Promotion of objects works. The objects that have been promoted + into old gen maintain whatever protocols and invariants are + assumed by Shenandoah GC. +2. That we can manage the transition from young GC to global GC. +3. That we can manage the transition from global GC to young GC. +4. That the transition from global GC to young GC + establishes all invariants required for correct operation of young + GC. + +Tasks: + +1. Distinguish between “global collection” for purposes of + maintaining support for non-generational GC and “global collection” + for purposes of supporting sequential interleaving of young GC + and global GC. +2. Implement the transition from global GC to young GC. +3. Initialize the remembered set for each consolidation heap region + of old gen to all CLEAN before allocating any GCLAB buffers within + the consolidation heap region. +4. At the start of global evacuation, select the collection set as + some combination of existing young regions and + old regions based on heuristics TBD. +5. During global collection, maintain old-gen GCLABs for all GC + threads and mutator threads. +6. During global collection, distinguish evacuation behavior + depending on whether an object to be evacuated resides within the + young collection set or the old collection set since + young objects are evacuated into young + consolidation regions and old objects are evacuated into old + consolidation regions; +7. Add minimal logging reports to describe behavior of young-gen and global + GC. +8. During global collection, perform maintenance on the contents of + each retired old GCLAB, where this maintenance consists of: + 1. Registering each object’s starting location and length with the + remembered set abstraction so that remembered set scanning can + quickly find the first object within each DIRTY card region, + 2. Updating the card table to reflect all references from this + object into young gen, + 3. Evacuating all objects directly referenced from this object + that reside within a collection set and that have not already been + evacuated, and + 4. Healing all pointers from within new replica objects residing in old + gen that refer to objects residing within the collection set. + 5. The last two items listed above are already performed by + traditional Shenandoah but can be merged with the implementation of + the other maintenance activities in order to perform all this work + in a consolidated pass. + +### Milestone 6: GC of young collections with Concurrent Marking (but not Collecting) of Old Gen + +__Demonstrated Concurrency Between Young-Gen and Old-Gen Activities__ + + ✓ denotes that this combination of activities is allowed. + + ✗ denotes that this combination of activities is disallowed. + +| | Old-Gen Mark | Old-Gen Evac | Old-Gen Idle | +|:--------------:|:--------------:|:-------------:|:------------:| +| Young Gen Mark | ✓ | NA | ✓ | +| Young Gen Evac | ✓ | NA | ✓ | +| Young Gen Idle | ✓ | NA | ✓ | + +This demonstration relies on GC log reports to show that marking of +old gen runs concurrently with marking of young gen. +Since the results of old-gen marking are not used to support old-gen +evacuation, this demonstration does not prove that old-gen marking +produces correct results. + +All pointers to old-gen memory that are discovered during scan of +young-gen memory are communicated to the old-gen concurrent mark +threads by inserting these pointer values into a SATB buffer as +keep-alive values. Every SATB buffer is post-processed both by a +young-gen GC thread and by an old-gen GC thread. + +Pointers from old-gen memory to young-gen memory that are discovered +during the marking of old-gen are ignored. + +At the start of young-gen concurrent marking, the remembered set is +scanned to detect all inter-generational references. + +The SATB write barrier remains enabled as long as either young-gen or +old-gen concurrent marking is active. + +Tasks: + +1. Each young GC thread has a dedicated SATB buffer into which it places + discovered references to old-gen memory. +2. SATB write barrier is left enabled as long as either young + or old marking is active. +3. SATB buffer is enlarged to 4096 entries. +4. SATB buffer compression is enhanced to deal with the mix of old + and young pointers. +5. SATB buffer processing is performed by both a young collection + thread and an old collection thread. Pointers to + old gen within the SATB buffer are marked and added to the old-gen + closure queues so that they can be subsequently scanned. +6. Certain GC threads (or work items) are dedicated to old GC + and others are dedicated to young GC. +7. Old-gen GC threads process the closure of previously marked + old-gen objects, scanning all references contained therein. +8. Old-gen GC threads add to the old-gen closures all old-gen objects + referenced by SATB buffers if those objects were not previously marked. + +### Milestone 7: Concurrent Young and Old Collections + + +__Demonstrated Concurrency Between Young-Gen and Old-Gen Activities__ + + ✓ denotes that this combination of activities is allowed. + + ✗ denotes that this combination of activities is disallowed. + +| | Old-Gen Mark | Old-Gen Evac | Old-Gen Idle | +|:--------------:|:--------------:|:-------------:|:------------:| +| Young Gen Mark | ✓ | ✗ | ✓ | +| Young Gen Evac | ✓ | ✓ | ✓ | +| Young Gen Idle | ✓ | ✗ | ✓ | + +In this demonstration, old-gen concurrent marking runs concurrently with all +phases of young-gen GC. Old-gen evacuation only runs while young-gen +evacuation is running. In the case that old-gen needs to evacuate so +much memory that doing so in a single uninterruptible batch would +significantly extend the duration of the young-gen evacuation phase, +the total old-gen evacuation workload is divided into multiple smaller +batches of evacuation work, each batch being processed concurrently +with a different young-gen evacuation cycle. + +The demonstration: + +1. Uses logging reports to describe the results of young + collection and old collection. +2. Shows for some “simple” workload (Heapothysis or + Extremem?) that generational GC provides performance benefits over + non-generational GC. + +Tasks + +1. Add minimal logging reports to describe behavior of old-gen + GC. +2. Decide which old regions comprise the old collection set. +3. Divide the old collection set into multiple collection subsets. +4. For each of the collection subsets + 1. Communicate the subset to young GC tasks to process these + evacuations when it begins its next evacuation cycle. + 2. Wait for young GC tasks to signal completion of the evacuation + cycle. + +## Proposed Future Milestones Not Yet Fully Planned + +### Milestone 8: Performance Improvements + +1. Remembered Set scanning sets cards to CLEAN if they are no longer + DIRTY. +2. Remembered Set scanning maintains and utilizes start-offset data + structure to quickly find the first object to be scanned within each + DIRTY card. +3. Remembered set scanning refrains from scanning the portions of + large objects and arrays that overlap card regions that are not + DIRTY. + +### Milestone 9: Fix Known Bugs + +We are aware of bugs in our existing card-marking implementation. + +### Milestone 10: Multiple Young-Gen Evacuations Process Old Collection Set + +### Milestone 11: Odd Objects (larger than 50% of TLAB/GCLAB size) + +By default, the promotion of such objects is handled by a +slower-than-normal path. Instead of allocating old gen from the +GCLAB, the mutator thread obtains memory for the copy by directly +accessing free lists. See existing code that does that already. + +### Micro Milestone 12: Collect and Report New Metrics + +### Micro Milestone 13: SATB-Based Remembered Set + +### Micro Milestone 14: Heuristic Pacing of Young Collection Frequency + +### Micro Milestone 15: Heuristic Pacing of Old Collection Frequency + +### Micro Milestone 16: Heuristic Sizing of Young and Old Sizes + +### Micro Milestone 17: Heuristic Adjustments of Tenuring Strategies + +### Micro Milestone 18: Overlap Evacuation of Cycle N with Marking of Cycle N+1 + +### Micro Milestone 19: Humongous Objects + +### Micro Milestone 20: Reference Processing + +### Micro Milestone 21: String Dedup + +### Micro Milestone 22: Degenerated GC + +### Micro Milesones TBD: Various Performance Improvements + diff --git a/src/hotspot/share/gc/shared/gcConfiguration.cpp b/src/hotspot/share/gc/shared/gcConfiguration.cpp index 195b64dcc3c94..fcc27e61db622 100644 --- a/src/hotspot/share/gc/shared/gcConfiguration.cpp +++ b/src/hotspot/share/gc/shared/gcConfiguration.cpp @@ -42,7 +42,14 @@ GCName GCConfiguration::young_collector() const { return ParallelScavenge; } - if (UseZGC || UseShenandoahGC) { + if (UseShenandoahGC) { + if (strcmp(ShenandoahGCMode, "generational") == 0) { + return Shenandoah; + } + return NA; + } + + if (UseZGC) { return NA; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 4908751c18609..b91904bf5e441 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -210,6 +210,8 @@ jint ShenandoahHeap::initialize() { "Cannot commit heap memory"); } + BarrierSet::set_barrier_set(new ShenandoahBarrierSet(this, _heap_region)); + // // After reserving the Java heap, create the card table, barriers, and workers, in dependency order // @@ -219,7 +221,6 @@ jint ShenandoahHeap::initialize() { rs = new ShenandoahDirectCardMarkRememberedSet(ShenandoahBarrierSet::barrier_set()->card_table(), card_count); _card_scan = new ShenandoahScanRemembered(rs); } - BarrierSet::set_barrier_set(new ShenandoahBarrierSet(this, _heap_region)); _workers = new ShenandoahWorkGang("Shenandoah GC Threads", _max_workers, /* are_GC_task_threads */ true, diff --git a/test/hotspot/jtreg/TEST.groups b/test/hotspot/jtreg/TEST.groups index 7efdfc779492b..11e193939212f 100644 --- a/test/hotspot/jtreg/TEST.groups +++ b/test/hotspot/jtreg/TEST.groups @@ -218,6 +218,15 @@ tier1_gc_gcbasher = \ gc/stress/gcbasher/TestGCBasherWithSerial.java \ gc/stress/gcbasher/TestGCBasherWithParallel.java +tier1_gc_shenandoah_generational = \ + gc/shenandoah/generational/ + +# No tier 2 tests for shenandoah_generational at this time +tier2_gc_shenandoah_generational = + +# No tier 3 tests for shenandoah_generational at this time +tier3_gc_shenandoah_generational = + tier1_gc_shenandoah = \ gc/shenandoah/options/ \ gc/shenandoah/compiler/ \ @@ -251,6 +260,7 @@ tier2_gc_shenandoah = \ -gc/stress/CriticalNativeStress.java \ -:tier1_gc_shenandoah +# include shenandoah generational tests in tier3 shenandoah tier3_gc_shenandoah = \ gc/stress/gcold/TestGCOldWithShenandoah.java \ gc/stress/gcbasher/TestGCBasherWithShenandoah.java \ @@ -258,6 +268,7 @@ tier3_gc_shenandoah = \ gc/stress/systemgc/TestSystemGCWithShenandoah.java \ gc/shenandoah/TestStringDedupStress.java \ gc/stress/CriticalNativeStress.java \ + :hotspot_gc_shenandoah_generational \ -:tier2_gc_shenandoah hotspot_gc_shenandoah = \ @@ -265,6 +276,11 @@ hotspot_gc_shenandoah = \ :tier2_gc_shenandoah \ :tier3_gc_shenandoah +hotspot_gc_shenandoah_generational = \ + :tier1_gc_shenandoah_generational \ + :tier2_gc_shenandoah_generational \ + :tier3_gc_shenandoah_generational + tier1_runtime = \ runtime/ \ -runtime/6626217/bug_21227.java \ diff --git a/test/hotspot/jtreg/gc/shenandoah/generational/TestCLIModeGenerational.java b/test/hotspot/jtreg/gc/shenandoah/generational/TestCLIModeGenerational.java new file mode 100644 index 0000000000000..beff1fb3113b3 --- /dev/null +++ b/test/hotspot/jtreg/gc/shenandoah/generational/TestCLIModeGenerational.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2021, Amazon, Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package gc.shenandoah.generational; + +import sun.hotspot.WhiteBox; + +/* + * @test TestCLIModeGenerational + * @requires vm.gc.Shenandoah + * @summary Test argument processing for -XX:+ShenandoahGCMode=generational. + * @library /testlibrary /test/lib / + * @build sun.hotspot.WhiteBox + * @run driver ClassFileInstaller sun.hotspot.WhiteBox + * @run main/othervm -Xbootclasspath/a:. + * -XX:+IgnoreUnrecognizedVMOptions + * -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * gc.shenandoah.generational.TestCLIModeGenerational + */ + +public class TestCLIModeGenerational { + + private static WhiteBox wb = WhiteBox.getWhiteBox(); + + public static void main(String args[]) throws Exception { + Boolean using_shenandoah = wb.getBooleanVMFlag("UseShenandoahGC"); + String gc_mode = wb.getStringVMFlag("ShenandoahGCMode"); + if (!using_shenandoah || !gc_mode.equals("generational")) + throw new IllegalStateException("Command-line options not honored!"); + } +} + diff --git a/test/hotspot/jtreg/gc/shenandoah/generational/TestSimpleGenerational.java b/test/hotspot/jtreg/gc/shenandoah/generational/TestSimpleGenerational.java new file mode 100644 index 0000000000000..c25ac586184fa --- /dev/null +++ b/test/hotspot/jtreg/gc/shenandoah/generational/TestSimpleGenerational.java @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2021, Amazon, Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +package gc.shenandoah.generational; + +import sun.hotspot.WhiteBox; +import java.util.Random; + +/* + * @test TestSimpleGenerational + * @requires vm.gc.Shenandoah + * @summary Confirm that card marking and remembered set scanning do not crash. + * @library /testlibrary /test/lib / + * @build sun.hotspot.WhiteBox + * @run driver ClassFileInstaller sun.hotspot.WhiteBox + * @run main/othervm -Xbootclasspath/a:. + * -XX:+IgnoreUnrecognizedVMOptions + * -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * gc.shenandoah.generational.TestSimpleGenerational + */ + +/* This used to be part of the run command, but caused problems. + * -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI + */ + +public class TestSimpleGenerational { + private static WhiteBox wb = WhiteBox.getWhiteBox(); + static private final int SeedForRandom = 46; + // Sequence of random numbers should end with same value + private static int ExpectedLastRandom = 272454100; + + + public static class Node { + static private final int NeighborCount = 5; + static private final int IntArraySize = 8; + static private Random random = new Random(SeedForRandom); + + private int val; + private Object field_o; + + // Each Node instance holds references to two "private" arrays. + // One array holds raw seething bits (primitive integers) and the + // holds references. + + private int[] field_ints; + private Node [] neighbors; + + public Node(int val) { + this.val = val; + this.field_o = new Object(); + this.field_ints = new int[IntArraySize]; + this.field_ints[0] = 0xca; + this.field_ints[1] = 0xfe; + this.field_ints[2] = 0xba; + this.field_ints[3] = 0xbe; + this.field_ints[4] = 0xba; + this.field_ints[5] = 0xad; + this.field_ints[6] = 0xba; + this.field_ints[7] = 0xbe; + + this.neighbors = new Node[NeighborCount]; + } + + public int value() { + return val; + } + + // Copy each neighbor of n into a new node's neighbor array. + // Then overwrite arbitrarily selected neighbor with newly allocated + // leaf node. + public static Node upheaval(Node n) { + int first_val = random.nextInt(); + if (first_val < 0) first_val = -first_val; + if (first_val < 0) first_val = 0; + Node result = new Node(first_val); + if (n != null) { + for (int i = 0; i < NeighborCount; i++) + result.neighbors[i] = n.neighbors[i]; + } + int second_val = random.nextInt(); + if (second_val < 0) second_val = -second_val; + if (second_val < 0) second_val = 0; + + int overwrite_index = first_val % NeighborCount; + result.neighbors[overwrite_index] = new Node(second_val); + return result; + } + } + + public static void main(String args[]) throws Exception { + Node n = null; + + if (!wb.getBooleanVMFlag("UseShenandoahGC") || + !wb.getStringVMFlag("ShenandoahGCMode").equals("generational")) + throw new IllegalStateException("Command-line options not honored!"); + + for (int count = 10000; count > 0; count--) { + n = Node.upheaval(n); + } + + if (n.value() != ExpectedLastRandom) + throw new IllegalStateException("Random number sequence ended badly!"); + + } + +} + From c78498984c109c3a50fb5c507477d075d72fc016 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Thu, 11 Feb 2021 10:25:53 -0800 Subject: [PATCH 010/254] Fix includes after merging header changes --- .../share/gc/shenandoah/shenandoahScanRemembered.cpp | 5 +---- .../share/gc/shenandoah/shenandoahScanRemembered.hpp | 6 ++---- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp index 19fa8ce42df0e..00f337269bd68 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp @@ -25,12 +25,9 @@ #include "precompiled.hpp" -#include "gc/shenandoah/shenandoahHeapRegion.hpp" + #include "gc/shenandoah/shenandoahScanRemembered.hpp" #include "gc/shenandoah/shenandoahHeap.hpp" -#include "gc/shenandoah/shenandoahHeap.inline.hpp" -#include "gc/shenandoah/shenandoahHeapRegion.hpp" -#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" ShenandoahDirectCardMarkRememberedSet::ShenandoahDirectCardMarkRememberedSet(CardTable *card_table, size_t total_card_count) { _heap = ShenandoahHeap::heap(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp index aa3d0895ec681..b59f0e62a3135 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp @@ -30,9 +30,7 @@ #include #include "memory/iterator.hpp" #include "gc/shenandoah/shenandoahCardTable.hpp" -#include "gc/shared/referenceDiscoverer.hpp" -#include "gc/shared/referencePolicy.hpp" -#include "gc/shenandoah/shenandoahReferenceProcessor.hpp" + // Terminology used within this source file: // @@ -213,7 +211,7 @@ // These limitations will be addressed in future enhancements to the // existing implementation. -class ReferenceProcessor; +class ShenandoahReferenceProcessor; class ShenandoahConcurrentMark; class ShenandoahHeap; class CardTable; From 1d26b9576da76e4c4a6f3d185976d56d30bf3742 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Thu, 11 Feb 2021 15:00:53 -0800 Subject: [PATCH 011/254] Restore lost generation parameters after merge --- .../shenandoah/shenandoahConcurrentMark.cpp | 111 +++++++++++++----- .../shenandoah/shenandoahConcurrentMark.hpp | 15 +-- .../gc/shenandoah/shenandoahControlThread.cpp | 2 +- .../gc/shenandoah/shenandoahGeneration.hpp | 9 +- .../share/gc/shenandoah/shenandoahHeap.cpp | 24 ++-- .../share/gc/shenandoah/shenandoahHeap.hpp | 10 +- .../share/gc/shenandoah/shenandoahMark.cpp | 70 ++++++----- .../share/gc/shenandoah/shenandoahMark.hpp | 9 +- .../gc/shenandoah/shenandoahMark.inline.hpp | 16 ++- .../shenandoahOopClosures.inline.hpp | 2 +- .../share/gc/shenandoah/shenandoahSTWMark.cpp | 4 +- 11 files changed, 173 insertions(+), 99 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp index 2289d71398b1d..ee852f51c86d3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp @@ -36,6 +36,7 @@ #include "gc/shenandoah/shenandoahBarrierSet.inline.hpp" #include "gc/shenandoah/shenandoahClosures.inline.hpp" #include "gc/shenandoah/shenandoahConcurrentMark.hpp" +#include "gc/shenandoah/shenandoahGeneration.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahMark.inline.hpp" #include "gc/shenandoah/shenandoahReferenceProcessor.hpp" @@ -81,6 +82,7 @@ class ShenandoahUpdateRootsTask : public AbstractGangTask { } }; +template class ShenandoahConcurrentMarkingTask : public AbstractGangTask { private: ShenandoahConcurrentMark* const _cm; @@ -98,7 +100,7 @@ class ShenandoahConcurrentMarkingTask : public AbstractGangTask { ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id); ShenandoahReferenceProcessor* rp = heap->ref_processor(); assert(rp != NULL, "need reference processor"); - _cm->mark_loop(worker_id, _terminator, rp, + _cm->mark_loop(GENERATION, worker_id, _terminator, rp, true, // cancellable ShenandoahStringDedup::is_enabled()); // perform string dedup } @@ -138,6 +140,7 @@ class ShenandoahSATBAndRemarkCodeRootsThreadsClosure : public ThreadClosure { } }; +template class ShenandoahFinalMarkingTask : public AbstractGangTask { private: ShenandoahConcurrentMark* _cm; @@ -159,7 +162,7 @@ class ShenandoahFinalMarkingTask : public AbstractGangTask { { ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id); - ShenandoahSATBBufferClosure cl(q); + ShenandoahSATBBufferClosure cl(q); SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set(); while (satb_mq_set.apply_closure_to_completed_buffer(&cl)) {} assert(!heap->has_forwarded_objects(), "Not expected"); @@ -173,7 +176,7 @@ class ShenandoahFinalMarkingTask : public AbstractGangTask { Threads::threads_do(&tc); } - _cm->mark_loop(worker_id, _terminator, rp, + _cm->mark_loop(GENERATION, worker_id, _terminator, rp, false, // not cancellable _dedup_string); @@ -181,6 +184,7 @@ class ShenandoahFinalMarkingTask : public AbstractGangTask { } }; +template class ShenandoahInitMarkRootsTask : public AbstractGangTask { private: ShenandoahRootScanner _root_scanner; @@ -198,7 +202,7 @@ class ShenandoahInitMarkRootsTask : public AbstractGangTask { assert(_task_queues->get_reserved() > worker_id, "Queue has not been reserved for worker id: %d", worker_id); ShenandoahObjToScanQueue* q = _task_queues->queue(worker_id); - ShenandoahInitMarkRootsClosure mark_cl(q); + ShenandoahInitMarkRootsClosure mark_cl(q); _root_scanner.roots_do(worker_id, &mark_cl); } }; @@ -206,7 +210,7 @@ class ShenandoahInitMarkRootsTask : public AbstractGangTask { ShenandoahConcurrentMark::ShenandoahConcurrentMark() : ShenandoahMark() {} -void ShenandoahConcurrentMark::mark_stw_roots() { +void ShenandoahConcurrentMark::mark_stw_roots(ShenandoahGeneration* generation) { assert(Thread::current()->is_VM_thread(), "Can only do this in VMThread"); assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint"); assert(!ShenandoahHeap::heap()->has_forwarded_objects(), "Not expected"); @@ -221,8 +225,20 @@ void ShenandoahConcurrentMark::mark_stw_roots() { TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats()); task_queues()->reserve(nworkers); - ShenandoahInitMarkRootsTask mark_roots(nworkers, task_queues()); - workers->run_task(&mark_roots); + switch (generation->generation_mode()) { + case YOUNG: { + ShenandoahInitMarkRootsTask mark_roots(nworkers, task_queues()); + workers->run_task(&mark_roots); + break; + } + case GLOBAL: { + ShenandoahInitMarkRootsTask mark_roots(nworkers, task_queues()); + workers->run_task(&mark_roots); + break; + } + default: + ShouldNotReachHere(); + } } void ShenandoahConcurrentMark::update_roots(ShenandoahPhaseTimings::Phase root_phase) { @@ -292,6 +308,7 @@ void ShenandoahConcurrentMark::update_thread_roots(ShenandoahPhaseTimings::Phase } // Mark concurrent roots during concurrent phases +template class ShenandoahMarkConcurrentRootsTask : public AbstractGangTask { private: ShenandoahConcurrentMark* _scm; @@ -309,11 +326,12 @@ class ShenandoahMarkConcurrentRootsTask : public AbstractGangTask { void work(uint worker_id); }; -ShenandoahMarkConcurrentRootsTask::ShenandoahMarkConcurrentRootsTask(ShenandoahConcurrentMark* scm, - ShenandoahObjToScanQueueSet* qs, - ShenandoahReferenceProcessor* rp, - ShenandoahPhaseTimings::Phase phase, - uint nworkers) : +template +ShenandoahMarkConcurrentRootsTask::ShenandoahMarkConcurrentRootsTask(ShenandoahConcurrentMark* scm, + ShenandoahObjToScanQueueSet* qs, + ShenandoahReferenceProcessor* rp, + ShenandoahPhaseTimings::Phase phase, + uint nworkers) : AbstractGangTask("Shenandoah Concurrent Mark Roots"), _root_scanner(nworkers, phase), _queue_set(qs), @@ -321,50 +339,75 @@ ShenandoahMarkConcurrentRootsTask::ShenandoahMarkConcurrentRootsTask(ShenandoahC assert(!ShenandoahHeap::heap()->has_forwarded_objects(), "Not expected"); } -void ShenandoahMarkConcurrentRootsTask::work(uint worker_id) { +template +void ShenandoahMarkConcurrentRootsTask::work(uint worker_id) { ShenandoahConcurrentWorkerSession worker_session(worker_id); ShenandoahObjToScanQueue* q = _queue_set->queue(worker_id); - ShenandoahMarkRefsClosure cl(q, _rp); + ShenandoahMarkRefsClosure cl(q, _rp); _root_scanner.roots_do(&cl, worker_id); } -void ShenandoahConcurrentMark::mark_concurrent_roots() { +void ShenandoahConcurrentMark::mark_concurrent_roots(ShenandoahGeneration* generation) { ShenandoahHeap* const heap = ShenandoahHeap::heap(); assert(!heap->has_forwarded_objects(), "Not expected"); WorkGang* workers = heap->workers(); ShenandoahReferenceProcessor* rp = heap->ref_processor(); task_queues()->reserve(workers->active_workers()); - ShenandoahMarkConcurrentRootsTask task(task_queues(), rp, ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers()); - - workers->run_task(&task); + switch (generation->generation_mode()) { + case YOUNG: { + ShenandoahMarkConcurrentRootsTask task(this, task_queues(), rp, ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers()); + workers->run_task(&task); + break; + } + case GLOBAL: { + ShenandoahMarkConcurrentRootsTask task(this, task_queues(), rp, ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers()); + workers->run_task(&task); + break; + } + default: + ShouldNotReachHere(); + } } -void ShenandoahConcurrentMark::concurrent_mark() { +void ShenandoahConcurrentMark::concurrent_mark(ShenandoahGeneration* generation) { ShenandoahHeap* const heap = ShenandoahHeap::heap(); WorkGang* workers = heap->workers(); uint nworkers = workers->active_workers(); task_queues()->reserve(nworkers); { - TaskTerminator terminator(nworkers, task_queues()); - ShenandoahConcurrentMarkingTask task(this, &terminator); - workers->run_task(&task); + switch (generation->generation_mode()) { + case YOUNG: { + TaskTerminator terminator(nworkers, task_queues()); + ShenandoahConcurrentMarkingTask task(this, &terminator); + workers->run_task(&task); + break; + } + case GLOBAL: { + TaskTerminator terminator(nworkers, task_queues()); + ShenandoahConcurrentMarkingTask task(this, &terminator); + workers->run_task(&task); + break; + } + default: + ShouldNotReachHere(); + } } assert(task_queues()->is_empty() || heap->cancelled_gc(), "Should be empty when not cancelled"); } -void ShenandoahConcurrentMark::finish_mark() { +void ShenandoahConcurrentMark::finish_mark(ShenandoahGeneration* generation) { assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint"); assert(Thread::current()->is_VM_thread(), "Must by VM Thread"); - finish_mark_work(); + finish_mark_work(generation); assert(task_queues()->is_empty(), "Should be empty"); TASKQUEUE_STATS_ONLY(task_queues()->print_taskqueue_stats()); TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats()); } -void ShenandoahConcurrentMark::finish_mark_work() { +void ShenandoahConcurrentMark::finish_mark_work(ShenandoahGeneration* generation) { // Finally mark everything else we've got in our queues during the previous steps. // It does two different things for concurrent vs. mark-compact GC: // - For concurrent GC, it starts with empty task queues, drains the remaining @@ -379,8 +422,22 @@ void ShenandoahConcurrentMark::finish_mark_work() { StrongRootsScope scope(nworkers); TaskTerminator terminator(nworkers, task_queues()); - ShenandoahFinalMarkingTask task(this, &terminator, ShenandoahStringDedup::is_enabled()); - heap->workers()->run_task(&task); + + switch (generation->generation_mode()) { + case YOUNG:{ + ShenandoahFinalMarkingTask task(this, &terminator, ShenandoahStringDedup::is_enabled()); + heap->workers()->run_task(&task); + break; + } + case GLOBAL:{ + ShenandoahFinalMarkingTask task(this, &terminator, ShenandoahStringDedup::is_enabled()); + heap->workers()->run_task(&task); + break; + } + default: + ShouldNotReachHere(); + } + assert(task_queues()->is_empty(), "Should be empty"); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp index e8e5cc6acc0b8..687e0f99a41c6 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp @@ -33,22 +33,23 @@ class ShenandoahStrDedupQueue; class ShenandoahReferenceProcessor; +class ShenandoahGeneration; class ShenandoahConcurrentMark: public ShenandoahMark { - friend class ShenandoahConcurrentMarkingTask; - friend class ShenandoahFinalMarkingTask; + template friend class ShenandoahConcurrentMarkingTask; + template friend class ShenandoahFinalMarkingTask; public: ShenandoahConcurrentMark(); // When concurrent stack processing is not supported - void mark_stw_roots(); - void mark_concurrent_roots(); + void mark_stw_roots(ShenandoahGeneration* generation); + void mark_concurrent_roots(ShenandoahGeneration* generation); // Concurrent mark - void concurrent_mark(); + void concurrent_mark(ShenandoahGeneration* generation); // Finish mark at a safepoint - void finish_mark(); + void finish_mark(ShenandoahGeneration* generation); static void cancel(); @@ -58,7 +59,7 @@ class ShenandoahConcurrentMark: public ShenandoahMark { static void update_thread_roots(ShenandoahPhaseTimings::Phase root_phase); private: - void finish_mark_work(); + void finish_mark_work(ShenandoahGeneration* generation); }; #endif // SHARE_GC_SHENANDOAH_SHENANDOAHCONCURRENTMARK_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index d19ca4e21e684..243f2a73ed9f1 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -407,7 +407,7 @@ void ShenandoahControlThread::service_concurrent_normal_cycle(GCCause::Cause cau heap->vmop_entry_init_mark(generation); // Concurrent mark roots - heap->entry_mark_roots(); + heap->entry_mark_roots(generation); if (check_cancellation_or_degen(ShenandoahHeap::_degenerated_outside_cycle)) return; // Continue concurrent mark diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index be41bec1344be..e1023e5c49be9 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -31,16 +31,13 @@ class ShenandoahGeneration : public CHeapObj { private: GenerationMode const _generation_mode; - ShenandoahConcurrentMark* const _scm; + public: - ShenandoahGeneration(GenerationMode generation_mode) : - _generation_mode(generation_mode), - _scm(new ShenandoahConcurrentMark(generation_mode)) { + explicit ShenandoahGeneration(GenerationMode generation_mode) : + _generation_mode(generation_mode) { } inline GenerationMode generation_mode() const { return _generation_mode; } - - inline ShenandoahConcurrentMark* concurrent_mark() const { return _scm; } }; #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHGENERATION_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 7b3a9f11b009e..c239a236aaec2 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -1635,7 +1635,7 @@ void ShenandoahHeap::op_init_mark(ShenandoahGeneration* generation) { OrderAccess::fence(); ShenandoahConcurrentMark mark; - mark.mark_stw_roots(); + mark.mark_stw_roots(generation); if (ShenandoahPacing) { pacer()->setup_for_mark(); @@ -1649,14 +1649,14 @@ void ShenandoahHeap::op_init_mark(ShenandoahGeneration* generation) { } } -void ShenandoahHeap::op_mark_roots() { +void ShenandoahHeap::op_mark_roots(ShenandoahGeneration* generation) { ShenandoahConcurrentMark mark; - mark.mark_concurrent_roots(); + mark.mark_concurrent_roots(generation); } -void ShenandoahHeap::op_mark() { +void ShenandoahHeap::op_mark(ShenandoahGeneration* generation) { ShenandoahConcurrentMark mark; - mark.concurrent_mark(); + mark.concurrent_mark(generation); } class ShenandoahFinalMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure { @@ -1711,7 +1711,7 @@ void ShenandoahHeap::op_final_mark(ShenandoahGeneration* generation) { assert(!has_forwarded_objects(), "No forwarded objects on this path"); if (!cancelled_gc()) { - finish_mark(); + finish_mark(generation); prepare_evacuation(); } else { // If this cycle was updating references, we need to keep the has_forwarded_objects @@ -1769,10 +1769,10 @@ void ShenandoahHeap::op_cleanup_complete() { } // Helpers -void ShenandoahHeap::finish_mark() { +void ShenandoahHeap::finish_mark(ShenandoahGeneration* generation) { assert(!cancelled_gc(), "Should not continue"); ShenandoahConcurrentMark mark; - mark.finish_mark(); + mark.finish_mark(generation); // Marking is completed, deactivate SATB barrier set_concurrent_mark_in_progress(false); mark_complete_marking_context(); @@ -2233,7 +2233,7 @@ void ShenandoahHeap::op_degenerated(ShenandoahDegenPoint point) { } case _degenerated_mark: if (point == _degenerated_mark) { - finish_mark(); + finish_mark(global_generation()); } prepare_evacuation(); @@ -3136,7 +3136,7 @@ void ShenandoahHeap::entry_degenerated(int point) { set_degenerated_gc_in_progress(false); } -void ShenandoahHeap::entry_mark_roots() { +void ShenandoahHeap::entry_mark_roots(ShenandoahGeneration* generation) { TraceCollectorStats tcs(monitoring_support()->concurrent_collection_counters()); const char* msg = "Concurrent marking roots"; @@ -3148,10 +3148,10 @@ void ShenandoahHeap::entry_mark_roots() { "concurrent marking roots"); try_inject_alloc_failure(); - op_mark_roots(); + op_mark_roots(generation); } -void ShenandoahHeap::entry_mark() { +void ShenandoahHeap::entry_mark(ShenandoahGeneration* generation) { TraceCollectorStats tcs(monitoring_support()->concurrent_collection_counters()); const char* msg = conc_mark_event_message(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index 7899695a888be..a1e225f96ced6 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -388,8 +388,8 @@ class ShenandoahHeap : public CollectedHeap { // Entry methods to normally concurrent GC operations. These set up logging, monitoring // for concurrent operation. void entry_reset(); - void entry_mark_roots(); - void entry_mark(); + void entry_mark_roots(ShenandoahGeneration* generation); + void entry_mark(ShenandoahGeneration* generation); void entry_weak_refs(); void entry_weak_roots(); void entry_class_unloading(); @@ -414,8 +414,8 @@ class ShenandoahHeap : public CollectedHeap { void op_degenerated_futile(); void op_reset(); - void op_mark_roots(); - void op_mark(); + void op_mark_roots(ShenandoahGeneration* generation); + void op_mark(ShenandoahGeneration* generation); void op_weak_refs(); void op_weak_roots(); void op_class_unloading(); @@ -439,7 +439,7 @@ class ShenandoahHeap : public CollectedHeap { const char* degen_event_message(ShenandoahDegenPoint point) const; // Helpers - void finish_mark(); + void finish_mark(ShenandoahGeneration* generation); void prepare_evacuation(); // ---------- GC subsystems diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp index 2dae7d6d9cb86..b857738df858d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp @@ -45,12 +45,6 @@ ShenandoahMarkRefsSuperClosure::ShenandoahMarkRefsSuperClosure(ShenandoahObjToSc _weak(false) { } -ShenandoahInitMarkRootsClosure::ShenandoahInitMarkRootsClosure(ShenandoahObjToScanQueue* q) : - _queue(q), - _heap(ShenandoahHeap::heap()), - _mark_context(_heap->marking_context()) { -} - ShenandoahMark::ShenandoahMark() : _task_queues(ShenandoahHeap::heap()->marking_context()->task_queues()) { } @@ -64,7 +58,7 @@ void ShenandoahMark::clear() { ShenandoahBarrierSet::satb_mark_queue_set().abandon_partial_marking(); } -template +template void ShenandoahMark::mark_loop_prework(uint w, TaskTerminator *t, ShenandoahReferenceProcessor *rp, bool strdedup) { ShenandoahObjToScanQueue* q = get_queue(w); @@ -77,37 +71,37 @@ void ShenandoahMark::mark_loop_prework(uint w, TaskTerminator *t, ShenandoahRefe if (heap->unload_classes()) { if (heap->has_forwarded_objects()) { if (strdedup) { - ShenandoahMarkUpdateRefsMetadataDedupClosure cl(q, rp); - mark_loop_work(&cl, ld, w, t); + ShenandoahMarkUpdateRefsMetadataDedupClosure cl(q, rp); + mark_loop_work, GENERATION, CANCELLABLE>(&cl, ld, w, t); } else { - ShenandoahMarkUpdateRefsMetadataClosure cl(q, rp); - mark_loop_work(&cl, ld, w, t); + ShenandoahMarkUpdateRefsMetadataClosure cl(q, rp); + mark_loop_work, GENERATION, CANCELLABLE>(&cl, ld, w, t); } } else { if (strdedup) { - ShenandoahMarkRefsMetadataDedupClosure cl(q, rp); - mark_loop_work(&cl, ld, w, t); + ShenandoahMarkRefsMetadataDedupClosure cl(q, rp); + mark_loop_work, GENERATION, CANCELLABLE>(&cl, ld, w, t); } else { - ShenandoahMarkRefsMetadataClosure cl(q, rp); - mark_loop_work(&cl, ld, w, t); + ShenandoahMarkRefsMetadataClosure cl(q, rp); + mark_loop_work, GENERATION, CANCELLABLE>(&cl, ld, w, t); } } } else { if (heap->has_forwarded_objects()) { if (strdedup) { - ShenandoahMarkUpdateRefsDedupClosure cl(q, rp); - mark_loop_work(&cl, ld, w, t); + ShenandoahMarkUpdateRefsDedupClosure cl(q, rp); + mark_loop_work, GENERATION, CANCELLABLE>(&cl, ld, w, t); } else { - ShenandoahMarkUpdateRefsClosure cl(q, rp); - mark_loop_work(&cl, ld, w, t); + ShenandoahMarkUpdateRefsClosure cl(q, rp); + mark_loop_work, GENERATION, CANCELLABLE>(&cl, ld, w, t); } } else { if (strdedup) { - ShenandoahMarkRefsDedupClosure cl(q, rp); - mark_loop_work(&cl, ld, w, t); + ShenandoahMarkRefsDedupClosure cl(q, rp); + mark_loop_work, GENERATION, CANCELLABLE>(&cl, ld, w, t); } else { - ShenandoahMarkRefsClosure cl(q, rp); - mark_loop_work(&cl, ld, w, t); + ShenandoahMarkRefsClosure cl(q, rp); + mark_loop_work, GENERATION, CANCELLABLE>(&cl, ld, w, t); } } } @@ -115,7 +109,7 @@ void ShenandoahMark::mark_loop_prework(uint w, TaskTerminator *t, ShenandoahRefe heap->flush_liveness_cache(w); } -template +template void ShenandoahMark::mark_loop_work(T* cl, ShenandoahLiveData* live_data, uint worker_id, TaskTerminator *terminator) { uintx stride = ShenandoahMarkLoopStride; @@ -154,7 +148,7 @@ void ShenandoahMark::mark_loop_work(T* cl, ShenandoahLiveData* live_data, uint w } q = get_queue(worker_id); - ShenandoahSATBBufferClosure drain_satb(q); + ShenandoahSATBBufferClosure drain_satb(q); SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set(); /* @@ -190,11 +184,27 @@ void ShenandoahMark::mark_loop_work(T* cl, ShenandoahLiveData* live_data, uint w } } -void ShenandoahMark::mark_loop(uint worker_id, TaskTerminator* terminator, ShenandoahReferenceProcessor *rp, +void ShenandoahMark::mark_loop(GenerationMode generation, uint worker_id, TaskTerminator* terminator, ShenandoahReferenceProcessor *rp, bool cancellable, bool strdedup) { - if (cancellable) { - mark_loop_prework(worker_id, terminator, rp, strdedup); - } else { - mark_loop_prework(worker_id, terminator, rp, strdedup); + switch (generation) { + case YOUNG: { + if (cancellable) { + mark_loop_prework(worker_id, terminator, rp, strdedup); + } else { + mark_loop_prework(worker_id, terminator, rp, strdedup); + } + break; + } + case GLOBAL: { + if (cancellable) { + mark_loop_prework(worker_id, terminator, rp, strdedup); + } else { + mark_loop_prework(worker_id, terminator, rp, strdedup); + } + break; + } + default: + ShouldNotReachHere(); + break; } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.hpp index c7a2f5753d931..51bc3f3fcba59 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMark.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.hpp @@ -32,6 +32,7 @@ class ShenandoahCMDrainMarkingStackClosure; +template class ShenandoahInitMarkRootsClosure : public OopClosure { private: ShenandoahObjToScanQueue* const _queue; @@ -61,7 +62,7 @@ class ShenandoahMark: public StackObj { ShenandoahMark(); public: - template + template static inline void mark_through_ref(T* p, ShenandoahHeap* heap, ShenandoahObjToScanQueue* q, ShenandoahMarkingContext* const mark_context, bool weak); static void clear(); @@ -83,14 +84,14 @@ class ShenandoahMark: public StackObj { inline void count_liveness(ShenandoahLiveData* live_data, oop obj); - template + template void mark_loop_work(T* cl, ShenandoahLiveData* live_data, uint worker_id, TaskTerminator *t); - template + template void mark_loop_prework(uint worker_id, TaskTerminator *terminator, ShenandoahReferenceProcessor *rp, bool strdedup); protected: - void mark_loop(uint worker_id, TaskTerminator* terminator, ShenandoahReferenceProcessor *rp, + void mark_loop(GenerationMode generation, uint worker_id, TaskTerminator* terminator, ShenandoahReferenceProcessor *rp, bool cancellable, bool strdedup); }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp index 711bcce4bcaba..d8bf07e828075 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp @@ -38,9 +38,17 @@ #include "runtime/prefetch.inline.hpp" #include "utilities/powerOfTwo.hpp" +template +ShenandoahInitMarkRootsClosure::ShenandoahInitMarkRootsClosure(ShenandoahObjToScanQueue* q) : + _queue(q), + _heap(ShenandoahHeap::heap()), + _mark_context(_heap->marking_context()) { +} + +template template -void ShenandoahInitMarkRootsClosure::do_oop_work(T* p) { - ShenandoahMark::mark_through_ref(p, _heap, _queue, _mark_context, false); +void ShenandoahInitMarkRootsClosure::do_oop_work(T* p) { + ShenandoahMark::mark_through_ref(p, _heap, _queue, _mark_context, false); } template @@ -236,12 +244,12 @@ class ShenandoahSATBBufferClosure : public SATBBufferClosure { void do_buffer_impl(void **buffer, size_t size) { for (size_t i = 0; i < size; ++i) { oop *p = (oop *) &buffer[i]; - ShenandoahMark::mark_through_ref(p, _heap, _queue, _mark_context, false); + ShenandoahMark::mark_through_ref(p, _heap, _queue, _mark_context, false); } } }; -template +template inline void ShenandoahMark::mark_through_ref(T *p, ShenandoahHeap* heap, ShenandoahObjToScanQueue* q, ShenandoahMarkingContext* const mark_context, bool weak) { T o = RawAccess<>::oop_load(p); if (!CompressedOops::is_null(o)) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp index c394b26784e14..cff4d84c7e36d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp @@ -30,7 +30,7 @@ template inline void ShenandoahMarkRefsSuperClosure::work(T *p) { - ShenandoahMark::mark_through_ref(p, _heap, _queue, _mark_context, _weak); + ShenandoahMark::mark_through_ref(p, _heap, _queue, _mark_context, _weak); } template diff --git a/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp index 1784d3d48690e..a5e1c0f0a1429 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp @@ -96,7 +96,7 @@ void ShenandoahSTWMark::mark() { } void ShenandoahSTWMark::mark_roots(uint worker_id) { - ShenandoahInitMarkRootsClosure init_mark(task_queues()->queue(worker_id)); + ShenandoahInitMarkRootsClosure init_mark(task_queues()->queue(worker_id)); _root_scanner.roots_do(&init_mark, worker_id); } @@ -105,7 +105,7 @@ void ShenandoahSTWMark::finish_mark(uint worker_id) { ShenandoahWorkerTimingsTracker timer(phase, ShenandoahPhaseTimings::ParallelMark, worker_id); ShenandoahReferenceProcessor* rp = ShenandoahHeap::heap()->ref_processor(); - mark_loop(worker_id, &_terminator, rp, + mark_loop(GLOBAL, worker_id, &_terminator, rp, false, // not cancellable ShenandoahStringDedup::is_enabled()); } From bf013e2e419eef330da32f38f6191cffacc38955 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Thu, 11 Feb 2021 15:11:41 -0800 Subject: [PATCH 012/254] Update reference to StoreValueBarrier flag --- .../share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp index e3c770e4afc2f..d981b266e3ebf 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp @@ -42,7 +42,7 @@ void ShenandoahGenerationalMode::initialize_flags() const { // Final configuration checks SHENANDOAH_CHECK_FLAG_SET(ShenandoahLoadRefBarrier); - SHENANDOAH_CHECK_FLAG_UNSET(ShenandoahStoreValEnqueueBarrier); + SHENANDOAH_CHECK_FLAG_UNSET(ShenandoahIUBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahSATBBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahCASBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahCloneBarrier); From fe871d5ab67677aa9e646a268caa7b254d96e5d0 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Thu, 11 Feb 2021 16:17:06 -0800 Subject: [PATCH 013/254] Pass generation through ShenandoahConcurrentGC (fixes merge) --- .../share/gc/shenandoah/shenandoahConcurrentGC.cpp | 14 ++++++++------ .../share/gc/shenandoah/shenandoahConcurrentGC.hpp | 9 ++++++--- .../gc/shenandoah/shenandoahControlThread.cpp | 2 +- .../gc/shenandoah/shenandoahDegeneratedGC.cpp | 2 +- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index 183002d56cc1d..b55fdaa24702c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -30,6 +30,7 @@ #include "gc/shenandoah/shenandoahCollectorPolicy.hpp" #include "gc/shenandoah/shenandoahConcurrentGC.hpp" #include "gc/shenandoah/shenandoahFreeSet.hpp" +#include "gc/shenandoah/shenandoahGeneration.hpp" #include "gc/shenandoah/shenandoahLock.hpp" #include "gc/shenandoah/shenandoahMark.inline.hpp" #include "gc/shenandoah/shenandoahMonitoringSupport.hpp" @@ -45,9 +46,10 @@ #include "runtime/vmThread.hpp" #include "utilities/events.hpp" -ShenandoahConcurrentGC::ShenandoahConcurrentGC() : +ShenandoahConcurrentGC::ShenandoahConcurrentGC(ShenandoahGeneration* generation) : _mark(), - _degen_point(ShenandoahDegenPoint::_degenerated_unset) { + _degen_point(ShenandoahDegenPoint::_degenerated_unset), + _generation(generation) { } ShenandoahGC::ShenandoahDegenPoint ShenandoahConcurrentGC::degen_point() const { @@ -488,7 +490,7 @@ void ShenandoahConcurrentGC::op_init_mark() { ShenandoahCodeRoots::arm_nmethods(); } - _mark.mark_stw_roots(); + _mark.mark_stw_roots(_generation); if (ShenandoahPacing) { heap->pacer()->setup_for_mark(); @@ -496,11 +498,11 @@ void ShenandoahConcurrentGC::op_init_mark() { } void ShenandoahConcurrentGC::op_mark_roots() { - _mark.mark_concurrent_roots(); + _mark.mark_concurrent_roots(_generation); } void ShenandoahConcurrentGC::op_mark() { - _mark.concurrent_mark(); + _mark.concurrent_mark(_generation); } void ShenandoahConcurrentGC::op_final_mark() { @@ -513,7 +515,7 @@ void ShenandoahConcurrentGC::op_final_mark() { } if (!heap->cancelled_gc()) { - _mark.finish_mark(); + _mark.finish_mark(_generation); assert(!heap->cancelled_gc(), "STW mark cannot OOM"); // Notify JVMTI that the tagmap table will need cleaning. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp index 36a61ca58f140..01ddb64c7a87e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp @@ -30,6 +30,8 @@ #include "gc/shenandoah/shenandoahGC.hpp" #include "gc/shenandoah/shenandoahHeap.hpp" +class ShenandoahGeneration; + class VM_ShenandoahInitMark; class VM_ShenandoahFinalMarkStartEvac; class VM_ShenandoahInitUpdateRefs; @@ -42,11 +44,12 @@ class ShenandoahConcurrentGC : public ShenandoahGC { friend class VM_ShenandoahFinalUpdateRefs; private: - ShenandoahConcurrentMark _mark; - ShenandoahDegenPoint _degen_point; + ShenandoahConcurrentMark _mark; + ShenandoahDegenPoint _degen_point; + ShenandoahGeneration* const _generation; public: - ShenandoahConcurrentGC(); + ShenandoahConcurrentGC(ShenandoahGeneration* generation); bool collect(GCCause::Cause cause); ShenandoahDegenPoint degen_point() const; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index c1380c1f98316..a63d5def98b2a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -401,7 +401,7 @@ void ShenandoahControlThread::service_concurrent_normal_cycle(GCCause::Cause cau TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); - ShenandoahConcurrentGC gc; + ShenandoahConcurrentGC gc(generation); if (gc.collect(cause)) { // Cycle is complete heap->heuristics()->record_success_concurrent(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp index 2619b1ec5635e..3e8d3f6e83140 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp @@ -227,7 +227,7 @@ void ShenandoahDegenGC::op_mark() { void ShenandoahDegenGC::op_finish_mark() { ShenandoahConcurrentMark mark; - mark.finish_mark(); + mark.finish_mark(ShenandoahHeap::heap()->global_generation()); } void ShenandoahDegenGC::op_prepare_evacuation() { From 0e47c8fe660004395db7f097ff32b03691b71ce7 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Thu, 11 Feb 2021 17:18:40 -0800 Subject: [PATCH 014/254] Fix merge fallout --- src/hotspot/share/gc/shenandoah/shenandoahMark.cpp | 5 ----- src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp | 5 ++--- src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp | 1 + 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp index edf0d10f504ac..4478b2934c1b4 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp @@ -44,11 +44,6 @@ ShenandoahMarkRefsSuperClosure::ShenandoahMarkRefsSuperClosure(ShenandoahObjToSc _weak(false) { } -ShenandoahInitMarkRootsClosure::ShenandoahInitMarkRootsClosure(ShenandoahObjToScanQueue* q) : - _queue(q), - _mark_context(ShenandoahHeap::heap()->marking_context()) { -} - ShenandoahMark::ShenandoahMark() : _task_queues(ShenandoahHeap::heap()->marking_context()->task_queues()) { } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp index eb2c9a40f705a..1e057fe73b248 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp @@ -42,8 +42,7 @@ template ShenandoahInitMarkRootsClosure::ShenandoahInitMarkRootsClosure(ShenandoahObjToScanQueue* q) : _queue(q), - _heap(ShenandoahHeap::heap()), - _mark_context(_heap->marking_context()) { + _mark_context(ShenandoahHeap::heap()->marking_context()) { } template @@ -259,7 +258,7 @@ inline void ShenandoahMark::mark_through_ref(T *p, ShenandoahObjToScanQueue* q, shenandoah_assert_not_forwarded(p, obj); shenandoah_assert_not_in_cset_except(p, obj, ShenandoahHeap::heap()->cancelled_gc()); - if (GENERATION != YOUNG || heap->is_in_young(obj)) { + if (GENERATION != YOUNG || ShenandoahHeap::heap()->is_in_young(obj)) { bool skip_live = false; bool marked; if (weak) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp index 74a233b6bf68f..1467bd85816dd 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp @@ -80,6 +80,7 @@ class ShenandoahMarkUpdateRefsSuperClosure : public ShenandoahMarkRefsSuperClosu }; }; +template class ShenandoahMarkUpdateRefsClosure : public ShenandoahMarkUpdateRefsSuperClosure { private: template From aaa3616800510fb06fa9fdd0ab0820c460fb8e35 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Fri, 12 Feb 2021 09:03:11 -0800 Subject: [PATCH 015/254] Remove reference to concurrent roots --- .../share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp index d981b266e3ebf..f41a50262cec8 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp @@ -23,7 +23,6 @@ */ #include "precompiled.hpp" -#include "gc/shenandoah/shenandoahConcurrentRoots.hpp" #include "gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp" #include "gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.hpp" #include "gc/shenandoah/heuristics/shenandoahCompactHeuristics.hpp" @@ -32,7 +31,7 @@ #include "runtime/java.hpp" void ShenandoahGenerationalMode::initialize_flags() const { - if (ShenandoahConcurrentRoots::can_do_concurrent_class_unloading()) { + if (ClassUnloading) { FLAG_SET_DEFAULT(ShenandoahSuspendibleWorkers, true); FLAG_SET_DEFAULT(VerifyBeforeExit, false); } From 4809d4724b5beb0ead67f162906f08eee614cd7c Mon Sep 17 00:00:00 2001 From: William Kemper Date: Fri, 12 Feb 2021 11:42:26 -0800 Subject: [PATCH 016/254] Correct include --- src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp index 00f337269bd68..80f57112af5e2 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp @@ -27,7 +27,7 @@ #include "precompiled.hpp" #include "gc/shenandoah/shenandoahScanRemembered.hpp" -#include "gc/shenandoah/shenandoahHeap.hpp" +#include "gc/shenandoah/shenandoahHeap.inline.hpp" ShenandoahDirectCardMarkRememberedSet::ShenandoahDirectCardMarkRememberedSet(CardTable *card_table, size_t total_card_count) { _heap = ShenandoahHeap::heap(); From e60b8a466a08c37a7d74effdc8d973ea48f2df51 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Tue, 16 Feb 2021 12:12:34 +0000 Subject: [PATCH 017/254] Mark concurrent roots as global when requested to do so, instead of young Reviewed-by: shade --- src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp index 6fb256a6cf8c4..89617a61f757e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp @@ -233,7 +233,7 @@ void ShenandoahConcurrentMark::mark_concurrent_roots(ShenandoahGeneration* gener break; } case GLOBAL: { - ShenandoahMarkConcurrentRootsTask task(this, task_queues(), rp, ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers()); + ShenandoahMarkConcurrentRootsTask task(this, task_queues(), rp, ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers()); workers->run_task(&task); break; } From c27335ddda37946932ddd658f1367b813232a9a1 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Tue, 16 Feb 2021 16:36:16 +0100 Subject: [PATCH 018/254] Resolve some merge conflicts --- .../shenandoahBarrierSetAssembler_x86.cpp | 6 +++--- .../shenandoah/c1/shenandoahBarrierSetC1.cpp | 12 ++++++------ .../shenandoah/c2/shenandoahBarrierSetC2.cpp | 18 +++++++++--------- .../shenandoah/shenandoahBarrierSet.inline.hpp | 10 +++++----- .../gc/shenandoah/shenandoahCardTable.hpp | 2 +- .../gc/shenandoah/shenandoahConcurrentMark.hpp | 2 ++ 6 files changed, 26 insertions(+), 24 deletions(-) diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp index 2358dc8cc724e..8c9829491ee7f 100644 --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp @@ -624,9 +624,9 @@ void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register o int dirty = CardTable::dirty_card_val(); if (UseCondCardMark) { Label L_already_dirty; - if (ct->scanned_concurrently()) { - __ membar(Assembler::StoreLoad); - } +// if (ct->scanned_concurrently()) { +// __ membar(Assembler::StoreLoad); +// } __ cmpb(card_addr, dirty); __ jcc(Assembler::equal, L_already_dirty); __ movb(card_addr, dirty); diff --git a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp index a4cc2a90c6e45..275924ea0742a 100644 --- a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp +++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp @@ -353,9 +353,9 @@ void ShenandoahBarrierSetC1::post_barrier(LIRAccess& access, LIR_OprDesc* addr, LIR_Opr dirty = LIR_OprFact::intConst(CardTable::dirty_card_val()); if (UseCondCardMark) { LIR_Opr cur_value = gen->new_register(T_INT); - if (ct->scanned_concurrently()) { - __ membar_storeload(); - } +// if (ct->scanned_concurrently()) { +// __ membar_storeload(); +// } __ move(card_addr, cur_value); LabelObj* L_already_dirty = new LabelObj(); @@ -364,9 +364,9 @@ void ShenandoahBarrierSetC1::post_barrier(LIRAccess& access, LIR_OprDesc* addr, __ move(dirty, card_addr); __ branch_destination(L_already_dirty->label()); } else { - if (ct->scanned_concurrently()) { - __ membar_storestore(); - } +// if (ct->scanned_concurrently()) { +// __ membar_storestore(); +// } __ move(dirty, card_addr); } } diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp index 9ac332acffdc7..58bf997502307 100644 --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp @@ -522,10 +522,10 @@ void ShenandoahBarrierSetC2::post_barrier(GraphKit* kit, Node* zero = __ ConI(0); // Dirty card value if (UseCondCardMark) { - if (ct->scanned_concurrently()) { - kit->insert_mem_bar(Op_MemBarVolatile, oop_store); - __ sync_kit(kit); - } +// if (ct->scanned_concurrently()) { +// kit->insert_mem_bar(Op_MemBarVolatile, oop_store); +// __ sync_kit(kit); +// } // The classic GC reference write barrier is typically implemented // as a store into the global card mark table. Unfortunately // unconditional stores can result in false sharing and excessive @@ -538,12 +538,12 @@ void ShenandoahBarrierSetC2::post_barrier(GraphKit* kit, } // Smash zero into card - if(!ct->scanned_concurrently()) { +// if(!ct->scanned_concurrently()) { __ store(__ ctrl(), card_adr, zero, T_BYTE, adr_type, MemNode::unordered); - } else { - // Specialized path for CM store barrier - __ storeCM(__ ctrl(), card_adr, zero, oop_store, adr_idx, T_BYTE, adr_type); - } +// } else { +// // Specialized path for CM store barrier +// __ storeCM(__ ctrl(), card_adr, zero, oop_store, adr_idx, T_BYTE, adr_type); +// } if (UseCondCardMark) { __ end_if(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp index c87751fe74499..88210bdda3851 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp @@ -196,12 +196,12 @@ template inline void ShenandoahBarrierSet::write_ref_field_post(T* field, oop newVal) { if (ShenandoahHeap::heap()->mode()->is_generational()) { volatile CardTable::CardValue* byte = card_table()->byte_for(field); - if (card_table()->scanned_concurrently()) { - // Perform a releasing store if the card table is scanned concurrently - Atomic::release_store(byte, CardTable::dirty_card_val()); - } else { + // if (card_table()->scanned_concurrently()) { + // // Perform a releasing store if the card table is scanned concurrently + // Atomic::release_store(byte, CardTable::dirty_card_val()); + // } else { *byte = CardTable::dirty_card_val(); - } + // } } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCardTable.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCardTable.hpp index 7a8c6c72c5039..a8e2cc32c926d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCardTable.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCardTable.hpp @@ -34,7 +34,7 @@ class ShenandoahCardTable: public CardTable { friend class VMStructs; public: - ShenandoahCardTable(MemRegion whole_heap): CardTable(whole_heap, /* conc_scan */ false) { } + ShenandoahCardTable(MemRegion whole_heap): CardTable(whole_heap) { } virtual void initialize(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp index 86e5dea06c6ac..0fcd6aece8b0e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp @@ -27,7 +27,9 @@ #include "gc/shenandoah/shenandoahMark.hpp" +template class ShenandoahConcurrentMarkingTask; +template class ShenandoahFinalMarkingTask; class ShenandoahGeneration; From 6ddc7de745351efe5b9c524ec636d6f748b921ed Mon Sep 17 00:00:00 2001 From: Aleksey Shipilev Date: Wed, 17 Feb 2021 10:16:51 +0000 Subject: [PATCH 019/254] Fix Windows build failures Reviewed-by: rkennke --- src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp index 80f57112af5e2..bc480672074f3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp @@ -33,7 +33,7 @@ ShenandoahDirectCardMarkRememberedSet::ShenandoahDirectCardMarkRememberedSet(Car _heap = ShenandoahHeap::heap(); _card_table = card_table; _total_card_count = total_card_count; - _cluster_count = (total_card_count / ShenandoahCardCluster::CardsPerCluster); + _cluster_count = uint32_t(total_card_count / ShenandoahCardCluster::CardsPerCluster); _card_shift = CardTable::card_shift; _byte_map = _card_table->byte_for_index(0); From a0f436c482853226e8f162ae421a1ac09b48b605 Mon Sep 17 00:00:00 2001 From: Aleksey Shipilev Date: Wed, 17 Feb 2021 10:40:06 +0000 Subject: [PATCH 020/254] Fix builds that do not have Shenandoah or other GCs Reviewed-by: rkennke --- src/hotspot/share/gc/shared/gcConfiguration.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/hotspot/share/gc/shared/gcConfiguration.cpp b/src/hotspot/share/gc/shared/gcConfiguration.cpp index 2f3c36f2b1216..0d96c85a5048a 100644 --- a/src/hotspot/share/gc/shared/gcConfiguration.cpp +++ b/src/hotspot/share/gc/shared/gcConfiguration.cpp @@ -32,6 +32,7 @@ #include "runtime/globals.hpp" #include "runtime/globals_extension.hpp" #include "utilities/debug.hpp" +#include "utilities/macros.hpp" GCName GCConfiguration::young_collector() const { if (UseG1GC) { @@ -43,9 +44,11 @@ GCName GCConfiguration::young_collector() const { } if (UseShenandoahGC) { +#if INCLUDE_SHENANDOAHGC if (strcmp(ShenandoahGCMode, "generational") == 0) { return Shenandoah; } +#endif return NA; } From 179c167a184325f581e133732c05c5d9bf96d760 Mon Sep 17 00:00:00 2001 From: Aleksey Shipilev Date: Thu, 18 Feb 2021 06:51:09 +0000 Subject: [PATCH 021/254] Fix Zero builds Reviewed-by: rkennke --- .../share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp index f41a50262cec8..0e9d70d859570 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp @@ -28,6 +28,9 @@ #include "gc/shenandoah/heuristics/shenandoahCompactHeuristics.hpp" #include "gc/shenandoah/heuristics/shenandoahStaticHeuristics.hpp" #include "gc/shenandoah/mode/shenandoahGenerationalMode.hpp" +#include "logging/log.hpp" +#include "logging/logTag.hpp" +#include "runtime/globals_extension.hpp" #include "runtime/java.hpp" void ShenandoahGenerationalMode::initialize_flags() const { From 4638e9783f2fb94c6d30b131f69872a24a5cf8c5 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Wed, 10 Mar 2021 20:37:00 +0000 Subject: [PATCH 022/254] Allow young collection to suspend marking in old generation Reviewed-by: rkennke --- .../c1/shenandoahBarrierSetC1_aarch64.cpp | 12 +- .../shenandoahBarrierSetAssembler_aarch64.cpp | 75 ++- .../shenandoahBarrierSetAssembler_aarch64.hpp | 6 + .../c1/shenandoahBarrierSetC1_x86.cpp | 2 + .../shenandoahBarrierSetAssembler_x86.cpp | 46 +- .../shenandoahBarrierSetAssembler_x86.hpp | 5 +- .../shenandoah/c2/shenandoahBarrierSetC2.cpp | 6 +- .../gc/shenandoah/c2/shenandoahSupport.cpp | 2 +- .../shenandoahAdaptiveHeuristics.cpp | 33 +- .../shenandoahAdaptiveHeuristics.hpp | 2 +- .../shenandoahAggressiveHeuristics.cpp | 2 +- .../shenandoahAggressiveHeuristics.hpp | 2 +- .../shenandoahCompactHeuristics.cpp | 15 +- .../shenandoahCompactHeuristics.hpp | 2 +- .../heuristics/shenandoahHeuristics.cpp | 32 +- .../heuristics/shenandoahHeuristics.hpp | 8 +- .../shenandoahPassiveHeuristics.hpp | 3 + .../heuristics/shenandoahStaticHeuristics.cpp | 13 +- .../heuristics/shenandoahStaticHeuristics.hpp | 2 +- .../mode/shenandoahGenerationalMode.cpp | 43 +- .../mode/shenandoahGenerationalMode.hpp | 15 +- .../gc/shenandoah/mode/shenandoahIUMode.cpp | 23 +- .../gc/shenandoah/mode/shenandoahIUMode.hpp | 2 - .../gc/shenandoah/mode/shenandoahMode.cpp | 50 ++ .../gc/shenandoah/mode/shenandoahMode.hpp | 5 +- .../shenandoah/mode/shenandoahPassiveMode.cpp | 6 +- .../shenandoah/mode/shenandoahPassiveMode.hpp | 3 +- .../gc/shenandoah/mode/shenandoahSATBMode.cpp | 23 +- .../gc/shenandoah/mode/shenandoahSATBMode.hpp | 1 - .../gc/shenandoah/shenandoahAllocRequest.hpp | 21 +- .../share/gc/shenandoah/shenandoahAsserts.cpp | 5 +- .../gc/shenandoah/shenandoahBarrierSet.cpp | 1 + .../gc/shenandoah/shenandoahBarrierSet.hpp | 2 +- .../shenandoahBarrierSet.inline.hpp | 27 +- .../shenandoahBarrierSetClone.inline.hpp | 6 +- .../gc/shenandoah/shenandoahCardTable.cpp | 19 + .../gc/shenandoah/shenandoahCardTable.hpp | 9 +- .../shenandoah/shenandoahClosures.inline.hpp | 6 +- .../gc/shenandoah/shenandoahConcurrentGC.cpp | 71 +-- .../gc/shenandoah/shenandoahConcurrentGC.hpp | 17 +- .../shenandoah/shenandoahConcurrentMark.cpp | 72 +-- .../shenandoah/shenandoahConcurrentMark.hpp | 13 +- .../gc/shenandoah/shenandoahControlThread.cpp | 412 ++++++++++++--- .../gc/shenandoah/shenandoahControlThread.hpp | 47 +- .../gc/shenandoah/shenandoahDegeneratedGC.cpp | 33 +- .../gc/shenandoah/shenandoahDegeneratedGC.hpp | 4 +- .../share/gc/shenandoah/shenandoahFreeSet.cpp | 52 +- .../share/gc/shenandoah/shenandoahFullGC.cpp | 36 +- .../gc/shenandoah/shenandoahGeneration.cpp | 276 ++++++++++ .../gc/shenandoah/shenandoahGeneration.hpp | 93 +++- .../shenandoah/shenandoahGlobalGeneration.cpp | 85 ++++ .../shenandoah/shenandoahGlobalGeneration.hpp | 21 +- .../share/gc/shenandoah/shenandoahHeap.cpp | 477 +++++++++--------- .../share/gc/shenandoah/shenandoahHeap.hpp | 83 ++- .../gc/shenandoah/shenandoahHeap.inline.hpp | 127 +++-- .../gc/shenandoah/shenandoahHeapRegion.cpp | 243 ++++++--- .../gc/shenandoah/shenandoahHeapRegion.hpp | 15 +- .../shenandoahHeapRegionCounters.cpp | 20 +- .../shenandoahHeapRegionCounters.hpp | 24 +- .../gc/shenandoah/shenandoahInitLogger.cpp | 3 +- .../share/gc/shenandoah/shenandoahMark.cpp | 78 ++- .../share/gc/shenandoah/shenandoahMark.hpp | 28 +- .../gc/shenandoah/shenandoahMark.inline.hpp | 70 ++- .../gc/shenandoah/shenandoahMarkClosures.cpp | 88 ++++ .../gc/shenandoah/shenandoahMarkClosures.hpp | 45 ++ .../shenandoah/shenandoahMarkingContext.cpp | 22 +- .../shenandoah/shenandoahMarkingContext.hpp | 11 +- .../shenandoahMarkingContext.inline.hpp | 8 + .../share/gc/shenandoah/shenandoahNMethod.cpp | 10 +- .../share/gc/shenandoah/shenandoahOldGC.cpp | 86 ++++ .../share/gc/shenandoah/shenandoahOldGC.hpp | 42 ++ .../gc/shenandoah/shenandoahOldGeneration.cpp | 161 ++++++ .../gc/shenandoah/shenandoahOldGeneration.hpp | 67 +++ .../gc/shenandoah/shenandoahOopClosures.hpp | 44 +- .../shenandoahOopClosures.inline.hpp | 2 +- .../share/gc/shenandoah/shenandoahPadding.hpp | 2 + .../gc/shenandoah/shenandoahPhaseTimings.hpp | 5 +- .../shenandoah/shenandoahRegulatorThread.cpp | 147 ++++++ .../shenandoah/shenandoahRegulatorThread.hpp | 87 ++++ .../share/gc/shenandoah/shenandoahSTWMark.cpp | 29 +- .../share/gc/shenandoah/shenandoahSTWMark.hpp | 3 +- .../shenandoah/shenandoahScanRemembered.cpp | 63 ++- .../shenandoah/shenandoahScanRemembered.hpp | 387 +++++++------- .../shenandoahScanRemembered.inline.hpp | 338 +++++++++---- .../shenandoah/shenandoahStackWatermark.cpp | 22 +- .../share/gc/shenandoah/shenandoahUnload.cpp | 2 +- .../share/gc/shenandoah/shenandoahUtils.cpp | 9 +- .../share/gc/shenandoah/shenandoahUtils.hpp | 4 +- .../gc/shenandoah/shenandoahVerifier.cpp | 18 +- .../shenandoah/shenandoahYoungGeneration.cpp | 139 +++++ .../shenandoah/shenandoahYoungGeneration.hpp | 30 +- .../gc/shenandoah/shenandoah_globals.hpp | 19 +- 92 files changed, 3674 insertions(+), 1151 deletions(-) create mode 100644 src/hotspot/share/gc/shenandoah/mode/shenandoahMode.cpp create mode 100644 src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp create mode 100644 src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp create mode 100644 src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.cpp create mode 100644 src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.hpp create mode 100644 src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp create mode 100644 src/hotspot/share/gc/shenandoah/shenandoahOldGC.hpp create mode 100644 src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp create mode 100644 src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp create mode 100644 src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.cpp create mode 100644 src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.hpp create mode 100644 src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp diff --git a/src/hotspot/cpu/aarch64/gc/shenandoah/c1/shenandoahBarrierSetC1_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shenandoah/c1/shenandoahBarrierSetC1_aarch64.cpp index e4f19f791f670..d87e93620a338 100644 --- a/src/hotspot/cpu/aarch64/gc/shenandoah/c1/shenandoahBarrierSetC1_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/gc/shenandoah/c1/shenandoahBarrierSetC1_aarch64.cpp @@ -87,10 +87,19 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess& access, LI LIR_Opr result = gen->new_register(T_INT); __ append(new LIR_OpShenandoahCompareAndSwap(addr, cmp_value.result(), new_value.result(), t1, t2, result)); + + post_barrier(access, access.resolved_addr(), new_value.result()); return result; } } - return BarrierSetC1::atomic_cmpxchg_at_resolved(access, cmp_value, new_value); + + LIR_Opr result = BarrierSetC1::atomic_cmpxchg_at_resolved(access, cmp_value, new_value); + + if (access.is_oop()) { + post_barrier(access, access.resolved_addr(), new_value.result()); + } + + return result; } LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess& access, LIRItem& value) { @@ -118,6 +127,7 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess& access, LIRIt pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), LIR_OprFact::illegalOpr, result /* pre_val */); } + post_barrier(access, access.resolved_addr(), result); } return result; diff --git a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp index 57a742f3fb9b7..ca20eb0eb7973 100644 --- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp @@ -31,6 +31,7 @@ #include "gc/shenandoah/shenandoahRuntime.hpp" #include "gc/shenandoah/shenandoahThreadLocalData.hpp" #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" +#include "gc/shenandoah/mode/shenandoahMode.hpp" #include "interpreter/interpreter.hpp" #include "interpreter/interp_masm.hpp" #include "runtime/sharedRuntime.hpp" @@ -60,7 +61,7 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec if (ShenandoahSATBBarrier && dest_uninitialized) { __ tbz(rscratch1, ShenandoahHeap::HAS_FORWARDED_BITPOS, done); } else { - __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING); + __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING); __ tst(rscratch1, rscratch2); __ br(Assembler::EQ, done); } @@ -77,6 +78,13 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec } } +void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, + Register start, Register count, Register tmp, RegSet saved_regs) { + if (is_oop) { + gen_write_ref_array_post_barrier(masm, decorators, start, count, tmp, saved_regs); + } +} + void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm, Register obj, Register pre_val, @@ -366,6 +374,35 @@ void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet d } } +void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj) { + if (!ShenandoahHeap::heap()->mode()->is_generational()) { + return; + } + + ShenandoahBarrierSet* ctbs = ShenandoahBarrierSet::barrier_set(); + CardTable* ct = ctbs->card_table(); + + __ lsr(obj, obj, CardTable::card_shift); + + assert(CardTable::dirty_card_val() == 0, "must be"); + + __ load_byte_map_base(rscratch1); + + if (UseCondCardMark) { + Label L_already_dirty; + __ membar(Assembler::StoreLoad); + __ ldrb(rscratch2, Address(obj, rscratch1)); + __ cbz(rscratch2, L_already_dirty); + __ strb(zr, Address(obj, rscratch1)); + __ bind(L_already_dirty); + } else { + // if (ct->scanned_concurrently()) { + // __ membar(Assembler::StoreStore); + // } + __ strb(zr, Address(obj, rscratch1)); + } +} + void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Address dst, Register val, Register tmp1, Register tmp2) { bool on_oop = is_reference_type(type); @@ -402,6 +439,7 @@ void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet __ mov(new_val, val); } BarrierSetAssembler::store_at(masm, decorators, type, Address(r3, 0), val, noreg, noreg); + store_check(masm, r3); } } @@ -586,6 +624,39 @@ void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, } } +void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, + Register start, Register count, Register scratch, RegSet saved_regs) { + if (!ShenandoahHeap::heap()->mode()->is_generational()) { + return; + } + + BarrierSet* bs = BarrierSet::barrier_set(); + CardTableBarrierSet* ctbs = barrier_set_cast(bs); + CardTable* ct = ctbs->card_table(); + + Label L_loop, L_done; + const Register end = count; + + __ cbz(count, L_done); // zero count - nothing to do + + __ lea(end, Address(start, count, Address::lsl(LogBytesPerHeapOop))); // end = start + count << LogBytesPerHeapOop + __ sub(end, end, BytesPerHeapOop); // last element address to make inclusive + __ lsr(start, start, CardTable::card_shift); + __ lsr(end, end, CardTable::card_shift); + __ sub(count, end, start); // number of bytes to copy + + __ load_byte_map_base(scratch); + __ add(start, start, scratch); + // if (ct->scanned_concurrently()) { + // __ membar(__ StoreStore); + // } + __ bind(L_loop); + __ strb(zr, Address(start, count)); + __ subs(count, count, 1); + __ br(Assembler::GE, L_loop); + __ bind(L_done); +} + #undef __ #ifdef COMPILER1 @@ -686,7 +757,7 @@ void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAss // Is marking still active? Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); __ ldrb(tmp, gc_state); - __ mov(rscratch2, ShenandoahHeap::MARKING); + __ mov(rscratch2, ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING); __ tst(tmp, rscratch2); __ br(Assembler::EQ, done); diff --git a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp index 1a20417f080ca..3745dafd20ce2 100644 --- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp @@ -54,10 +54,14 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler { bool tosca_live, bool expand_call); + void store_check(MacroAssembler* masm, Register obj); + void resolve_forward_pointer(MacroAssembler* masm, Register dst, Register tmp = noreg); void resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp = noreg); void load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr, DecoratorSet decorators); + void gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, Register start, Register count, Register scratch, RegSet saved_regs); + public: void iu_barrier(MacroAssembler* masm, Register dst, Register tmp); @@ -71,6 +75,8 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler { virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, Register src, Register dst, Register count, RegSet saved_regs); + virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop, + Register start, Register count, Register tmp, RegSet saved_regs); virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register dst, Address src, Register tmp1, Register tmp_thread); virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, diff --git a/src/hotspot/cpu/x86/gc/shenandoah/c1/shenandoahBarrierSetC1_x86.cpp b/src/hotspot/cpu/x86/gc/shenandoah/c1/shenandoahBarrierSetC1_x86.cpp index 7943781865224..9ef4cb6f858d7 100644 --- a/src/hotspot/cpu/x86/gc/shenandoah/c1/shenandoahBarrierSetC1_x86.cpp +++ b/src/hotspot/cpu/x86/gc/shenandoah/c1/shenandoahBarrierSetC1_x86.cpp @@ -87,6 +87,8 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess& access, LI LIR_Opr result = gen->new_register(T_INT); __ append(new LIR_OpShenandoahCompareAndSwap(addr, cmp_value.result(), new_value.result(), t1, t2, result)); + + post_barrier(access, access.resolved_addr(), new_value.result()); return result; } } diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp index 8c9829491ee7f..6014847857ef5 100644 --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp @@ -152,7 +152,7 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec if (ShenandoahSATBBarrier && dest_uninitialized) { flags = ShenandoahHeap::HAS_FORWARDED; } else { - flags = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING; + flags = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING; } __ testb(gc_state, flags); __ jcc(Assembler::zero, done); @@ -182,6 +182,39 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec } +void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Register src, Register dst, Register count) { + bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0; + bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0; + bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops); +#ifdef _LP64 + Register tmp = rscratch1; +#else + Register tmp = rax; +#endif + +if (is_reference_type(type)) { +#ifdef _LP64 +#if COMPILER2_OR_JVMCI + if (VM_Version::supports_avx512vlbw() && MaxVectorSize >= 32 && !checkcast) { + if (!obj_int) { + // Save count for barrier + count = r11; + } else if (disjoint) { + // Use the saved dst in the disjoint case + dst = r11; + } + } +# endif +#else + if (disjoint) { + __ mov(dst, rdx); // restore 'to' + } +#endif + gen_write_ref_array_post_barrier(masm, decorators, dst, count, tmp); + } +} + void ShenandoahBarrierSetAssembler::shenandoah_write_barrier_pre(MacroAssembler* masm, Register obj, Register pre_val, @@ -225,7 +258,7 @@ void ShenandoahBarrierSetAssembler::satb_write_barrier_pre(MacroAssembler* masm, Address buffer(thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset())); Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); - __ testb(gc_state, ShenandoahHeap::MARKING); + __ testb(gc_state, ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING); __ jcc(Assembler::zero, done); // Do we need to load the previous value? @@ -587,16 +620,15 @@ void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet d } } -void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj, Address dst) { +void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj) { if (!ShenandoahHeap::heap()->mode()->is_generational()) { return; } // Does a store check for the oop in register obj. The content of // register obj is destroyed afterwards. - BarrierSet* bs = BarrierSet::barrier_set(); - ShenandoahBarrierSet* ctbs = barrier_set_cast(bs); + ShenandoahBarrierSet* ctbs = ShenandoahBarrierSet::barrier_set(); CardTable* ct = ctbs->card_table(); __ shrptr(obj, CardTable::card_shift); @@ -679,7 +711,7 @@ void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet } else { iu_barrier(masm, val, tmp3); BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg); - store_check(masm, tmp1, dst); + store_check(masm, tmp1); } NOT_LP64(imasm->restore_bcp()); } else { @@ -1043,7 +1075,7 @@ void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAss // Is SATB still active? Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); - __ testb(gc_state, ShenandoahHeap::MARKING); + __ testb(gc_state, ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING); __ jcc(Assembler::zero, done); // Can we store original value in the thread's buffer? diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp index 42a42452fc7ef..1587fb4a46b2e 100644 --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp @@ -58,7 +58,8 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler { void iu_barrier_impl(MacroAssembler* masm, Register dst, Register tmp); - void store_check(MacroAssembler* masm, Register obj, Address dst); + void store_check(MacroAssembler* masm, Register obj); + void gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, Register addr, Register count, Register tmp); public: @@ -77,6 +78,8 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler { bool exchange, Register tmp1, Register tmp2); virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count); + virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Register src, Register dst, Register count); virtual void load_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register dst, Address src, Register tmp1, Register tmp_thread); virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp index 58bf997502307..c50a03a35084d 100644 --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp @@ -242,7 +242,7 @@ void ShenandoahBarrierSetC2::satb_write_barrier_pre(GraphKit* kit, Node* marking; Node* gc_state = __ AddP(no_base, tls, __ ConX(in_bytes(ShenandoahThreadLocalData::gc_state_offset()))); Node* ld = __ load(__ ctrl(), gc_state, TypeInt::BYTE, T_BYTE, Compile::AliasIdxRaw); - marking = __ AndI(ld, __ ConI(ShenandoahHeap::MARKING)); + marking = __ AndI(ld, __ ConI(ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING)); assert(ShenandoahBarrierC2Support::is_gc_state_load(ld), "Should match the shape"); // if (!marking) @@ -323,7 +323,7 @@ bool ShenandoahBarrierSetC2::is_shenandoah_marking_if(PhaseTransform *phase, Nod cmpx->is_Cmp() && cmpx->in(2) == phase->intcon(0) && is_shenandoah_state_load(cmpx->in(1)->in(1)) && cmpx->in(1)->in(2)->is_Con() && - cmpx->in(1)->in(2) == phase->intcon(ShenandoahHeap::MARKING)) { + cmpx->in(1)->in(2) == phase->intcon(ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING)) { return true; } @@ -950,7 +950,7 @@ void ShenandoahBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCo Node* gc_state = phase->transform_later(new LoadBNode(ctrl, mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered)); int flags = ShenandoahHeap::HAS_FORWARDED; if (ShenandoahIUBarrier) { - flags |= ShenandoahHeap::MARKING; + flags |= ShenandoahHeap::YOUNG_MARKING; } Node* stable_and = phase->transform_later(new AndINode(gc_state, phase->igvn().intcon(flags))); Node* stable_cmp = phase->transform_later(new CmpINode(stable_and, phase->igvn().zerocon(T_INT))); diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp index 8b7e5c10f6de3..4f83195c3d28b 100644 --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp @@ -1500,7 +1500,7 @@ void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) { Node* phi2 = PhiNode::make(region2, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM); // Stable path. - test_gc_state(ctrl, raw_mem, heap_stable_ctrl, phase, ShenandoahHeap::MARKING); + test_gc_state(ctrl, raw_mem, heap_stable_ctrl, phase, (ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING)); region->init_req(_heap_stable, heap_stable_ctrl); phi->init_req(_heap_stable, raw_mem); diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp index 819f1e8d74e24..ef8227ee4bf4e 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp @@ -27,7 +27,7 @@ #include "gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp" #include "gc/shenandoah/shenandoahCollectionSet.hpp" #include "gc/shenandoah/shenandoahFreeSet.hpp" -#include "gc/shenandoah/shenandoahHeap.inline.hpp" +#include "gc/shenandoah/shenandoahGeneration.hpp" #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" #include "logging/log.hpp" #include "logging/logTag.hpp" @@ -54,8 +54,8 @@ const double ShenandoahAdaptiveHeuristics::HIGHEST_EXPECTED_AVAILABLE_AT_END = 0 const double ShenandoahAdaptiveHeuristics::MINIMUM_CONFIDENCE = 0.319; // 25% const double ShenandoahAdaptiveHeuristics::MAXIMUM_CONFIDENCE = 3.291; // 99.9% -ShenandoahAdaptiveHeuristics::ShenandoahAdaptiveHeuristics() : - ShenandoahHeuristics(), +ShenandoahAdaptiveHeuristics::ShenandoahAdaptiveHeuristics(ShenandoahGeneration* generation) : + ShenandoahHeuristics(generation), _margin_of_error_sd(ShenandoahAdaptiveInitialConfidence), _spike_threshold_sd(ShenandoahAdaptiveInitialSpikeThreshold), _last_trigger(OTHER) { } @@ -84,7 +84,7 @@ void ShenandoahAdaptiveHeuristics::choose_collection_set_from_regiondata(Shenand // we hit max_cset. When max_cset is hit, we terminate the cset selection. Note that in this scheme, // ShenandoahGarbageThreshold is the soft threshold which would be ignored until min_garbage is hit. - size_t capacity = ShenandoahHeap::heap()->soft_max_capacity(); + size_t capacity = _generation->soft_max_capacity(); size_t max_cset = (size_t)((1.0 * capacity / 100 * ShenandoahEvacReserve) / ShenandoahEvacWaste); size_t free_target = (capacity / 100 * ShenandoahMinFreeThreshold) + max_cset; size_t min_garbage = (free_target > actual_free ? (free_target - actual_free) : 0); @@ -196,11 +196,10 @@ static double saturate(double value, double min, double max) { } bool ShenandoahAdaptiveHeuristics::should_start_gc() { - ShenandoahHeap* heap = ShenandoahHeap::heap(); - size_t max_capacity = heap->max_capacity(); - size_t capacity = heap->soft_max_capacity(); - size_t available = heap->free_set()->available(); - size_t allocated = heap->bytes_allocated_since_gc_start(); + size_t max_capacity = _generation->max_capacity(); + size_t capacity = _generation->soft_max_capacity(); + size_t available = _generation->available(); + size_t allocated = _generation->bytes_allocated_since_gc_start(); // Make sure the code below treats available without the soft tail. size_t soft_tail = max_capacity - capacity; @@ -212,18 +211,20 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { size_t min_threshold = capacity / 100 * ShenandoahMinFreeThreshold; if (available < min_threshold) { - log_info(gc)("Trigger: Free (" SIZE_FORMAT "%s) is below minimum threshold (" SIZE_FORMAT "%s)", + log_info(gc)("Trigger (%s): Free (" SIZE_FORMAT "%s) is below minimum threshold (" SIZE_FORMAT "%s)", + _generation->name(), byte_size_in_proper_unit(available), proper_unit_for_byte_size(available), byte_size_in_proper_unit(min_threshold), proper_unit_for_byte_size(min_threshold)); return true; } + // Check if are need to learn a bit about the application const size_t max_learn = ShenandoahLearningSteps; if (_gc_times_learned < max_learn) { size_t init_threshold = capacity / 100 * ShenandoahInitFreeThreshold; if (available < init_threshold) { - log_info(gc)("Trigger: Learning " SIZE_FORMAT " of " SIZE_FORMAT ". Free (" SIZE_FORMAT "%s) is below initial threshold (" SIZE_FORMAT "%s)", - _gc_times_learned + 1, max_learn, + log_info(gc)("Trigger (%s): Learning " SIZE_FORMAT " of " SIZE_FORMAT ". Free (" SIZE_FORMAT "%s) is below initial threshold (" SIZE_FORMAT "%s)", + _generation->name(), _gc_times_learned + 1, max_learn, byte_size_in_proper_unit(available), proper_unit_for_byte_size(available), byte_size_in_proper_unit(init_threshold), proper_unit_for_byte_size(init_threshold)); return true; @@ -244,8 +245,8 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { double avg_cycle_time = _gc_time_history->davg() + (_margin_of_error_sd * _gc_time_history->dsd()); double avg_alloc_rate = _allocation_rate.upper_bound(_margin_of_error_sd); if (avg_cycle_time > allocation_headroom / avg_alloc_rate) { - log_info(gc)("Trigger: Average GC time (%.2f ms) is above the time for average allocation rate (%.0f %sB/s) to deplete free headroom (" SIZE_FORMAT "%s) (margin of error = %.2f)", - avg_cycle_time * 1000, + log_info(gc)("Trigger (%s): Average GC time (%.2f ms) is above the time for average allocation rate (%.0f %sB/s) to deplete free headroom (" SIZE_FORMAT "%s) (margin of error = %.2f)", + _generation->name(), avg_cycle_time * 1000, byte_size_in_proper_unit(avg_alloc_rate), proper_unit_for_byte_size(avg_alloc_rate), byte_size_in_proper_unit(allocation_headroom), proper_unit_for_byte_size(allocation_headroom), _margin_of_error_sd); @@ -262,8 +263,8 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { bool is_spiking = _allocation_rate.is_spiking(rate, _spike_threshold_sd); if (is_spiking && avg_cycle_time > allocation_headroom / rate) { - log_info(gc)("Trigger: Average GC time (%.2f ms) is above the time for instantaneous allocation rate (%.0f %sB/s) to deplete free headroom (" SIZE_FORMAT "%s) (spike threshold = %.2f)", - avg_cycle_time * 1000, + log_info(gc)("Trigger (%s): Average GC time (%.2f ms) is above the time for instantaneous allocation rate (%.0f %sB/s) to deplete free headroom (" SIZE_FORMAT "%s) (spike threshold = %.2f)", + _generation->name(), avg_cycle_time * 1000, byte_size_in_proper_unit(rate), proper_unit_for_byte_size(rate), byte_size_in_proper_unit(allocation_headroom), proper_unit_for_byte_size(allocation_headroom), _spike_threshold_sd); diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp index a1a0e6321fafe..babedaaa24274 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp @@ -53,7 +53,7 @@ class ShenandoahAllocationRate : public CHeapObj { class ShenandoahAdaptiveHeuristics : public ShenandoahHeuristics { public: - ShenandoahAdaptiveHeuristics(); + ShenandoahAdaptiveHeuristics(ShenandoahGeneration* generation); virtual ~ShenandoahAdaptiveHeuristics(); diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp index 4ba3a0315b7c5..962c39c4fba57 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp @@ -31,7 +31,7 @@ #include "logging/logTag.hpp" #include "runtime/os.hpp" -ShenandoahAggressiveHeuristics::ShenandoahAggressiveHeuristics() : ShenandoahHeuristics() { +ShenandoahAggressiveHeuristics::ShenandoahAggressiveHeuristics(ShenandoahGeneration* generation) : ShenandoahHeuristics(generation) { // Do not shortcut evacuation SHENANDOAH_ERGO_OVERRIDE_DEFAULT(ShenandoahImmediateThreshold, 100); diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.hpp index e90d2da734730..4f11d96377e47 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.hpp @@ -29,7 +29,7 @@ class ShenandoahAggressiveHeuristics : public ShenandoahHeuristics { public: - ShenandoahAggressiveHeuristics(); + ShenandoahAggressiveHeuristics(ShenandoahGeneration* generation); virtual void choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset, RegionData* data, size_t size, diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp index 514a425f58b11..0591379357f7e 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp @@ -27,12 +27,13 @@ #include "gc/shenandoah/shenandoahCollectionSet.hpp" #include "gc/shenandoah/heuristics/shenandoahCompactHeuristics.hpp" #include "gc/shenandoah/shenandoahFreeSet.hpp" -#include "gc/shenandoah/shenandoahHeap.inline.hpp" +#include "gc/shenandoah/shenandoahGeneration.hpp" #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" #include "logging/log.hpp" #include "logging/logTag.hpp" -ShenandoahCompactHeuristics::ShenandoahCompactHeuristics() : ShenandoahHeuristics() { +ShenandoahCompactHeuristics::ShenandoahCompactHeuristics(ShenandoahGeneration* generation) : + ShenandoahHeuristics(generation) { SHENANDOAH_ERGO_ENABLE_FLAG(ExplicitGCInvokesConcurrent); SHENANDOAH_ERGO_ENABLE_FLAG(ShenandoahImplicitGCInvokesConcurrent); SHENANDOAH_ERGO_ENABLE_FLAG(ShenandoahUncommit); @@ -45,11 +46,9 @@ ShenandoahCompactHeuristics::ShenandoahCompactHeuristics() : ShenandoahHeuristic } bool ShenandoahCompactHeuristics::should_start_gc() { - ShenandoahHeap* heap = ShenandoahHeap::heap(); - - size_t max_capacity = heap->max_capacity(); - size_t capacity = heap->soft_max_capacity(); - size_t available = heap->free_set()->available(); + size_t max_capacity = _generation->max_capacity(); + size_t capacity = _generation->soft_max_capacity(); + size_t available = _generation->available(); // Make sure the code below treats available without the soft tail. size_t soft_tail = max_capacity - capacity; @@ -65,7 +64,7 @@ bool ShenandoahCompactHeuristics::should_start_gc() { return true; } - size_t bytes_allocated = heap->bytes_allocated_since_gc_start(); + size_t bytes_allocated = _generation->bytes_allocated_since_gc_start(); if (bytes_allocated > threshold_bytes_allocated) { log_info(gc)("Trigger: Allocated since last cycle (" SIZE_FORMAT "%s) is larger than allocation threshold (" SIZE_FORMAT "%s)", byte_size_in_proper_unit(bytes_allocated), proper_unit_for_byte_size(bytes_allocated), diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.hpp index 0fd9641d4a274..eb877921a7483 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.hpp @@ -29,7 +29,7 @@ class ShenandoahCompactHeuristics : public ShenandoahHeuristics { public: - ShenandoahCompactHeuristics(); + ShenandoahCompactHeuristics(ShenandoahGeneration* generation); virtual bool should_start_gc(); diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp index 009cf11034d40..43a35bf1361f9 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp @@ -24,12 +24,15 @@ #include "precompiled.hpp" #include "gc/shared/gcCause.hpp" +#include "gc/shenandoah/shenandoahAllocRequest.hpp" #include "gc/shenandoah/shenandoahCollectionSet.inline.hpp" #include "gc/shenandoah/shenandoahCollectorPolicy.hpp" +#include "gc/shenandoah/shenandoahGeneration.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp" #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" +#include "gc/shenandoah/mode/shenandoahMode.hpp" #include "logging/log.hpp" #include "logging/logTag.hpp" #include "runtime/globals_extension.hpp" @@ -42,7 +45,7 @@ int ShenandoahHeuristics::compare_by_garbage(RegionData a, RegionData b) { else return 0; } -ShenandoahHeuristics::ShenandoahHeuristics() : +ShenandoahHeuristics::ShenandoahHeuristics(ShenandoahGeneration* generation) : _region_data(NULL), _degenerated_cycles_in_a_row(0), _successful_cycles_in_a_row(0), @@ -51,7 +54,8 @@ ShenandoahHeuristics::ShenandoahHeuristics() : _gc_times_learned(0), _gc_time_penalties(0), _gc_time_history(new TruncatedSeq(10, ShenandoahAdaptiveDecayFactor)), - _metaspace_oom() + _metaspace_oom(), + _generation(generation) { // No unloading during concurrent mark? Communicate that to heuristics if (!ClassUnloadingWithConcurrentMark) { @@ -92,11 +96,11 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec size_t free = 0; size_t free_regions = 0; - ShenandoahMarkingContext* const ctx = heap->complete_marking_context(); + ShenandoahMarkingContext* const ctx = _generation->complete_marking_context(); for (size_t i = 0; i < num_regions; i++) { ShenandoahHeapRegion* region = heap->get_region(i); - if (heap->mode()->is_generational() && region->affiliation() != ShenandoahRegionAffiliation::YOUNG_GENERATION) { + if (!in_generation(region)) { continue; } @@ -107,18 +111,24 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec free_regions++; free += ShenandoahHeapRegion::region_size_bytes(); } else if (region->is_regular()) { - if (!region->has_live()) { + if (!region->has_live() && !heap->mode()->is_generational()) { // We can recycle it right away and put it in the free set. immediate_regions++; immediate_garbage += garbage; region->make_trash_immediate(); - } else { + } else if (_generation->generation_mode() != OLD) { + // HEY! At this stage in development our concurrent old + // marking does NOT complete the subsequent phases of the collection + // and we don't want regions stuck in the 'in_cset' state because + // various asserts will trip. + // This is our candidate for later consideration. candidates[cand_idx]._region = region; candidates[cand_idx]._garbage = garbage; cand_idx++; } } else if (region->is_humongous_start()) { + // Reclaim humongous regions here, and count them as the immediate garbage #ifdef ASSERT bool reg_live = region->has_live(); @@ -196,8 +206,8 @@ bool ShenandoahHeuristics::should_start_gc() { if (ShenandoahGuaranteedGCInterval > 0) { double last_time_ms = (os::elapsedTime() - _last_cycle_end) * 1000; if (last_time_ms > ShenandoahGuaranteedGCInterval) { - log_info(gc)("Trigger: Time since last GC (%.0f ms) is larger than guaranteed interval (" UINTX_FORMAT " ms)", - last_time_ms, ShenandoahGuaranteedGCInterval); + log_info(gc)("Trigger (%s): Time since last GC (%.0f ms) is larger than guaranteed interval (" UINTX_FORMAT " ms)", + _generation->name(), last_time_ms, ShenandoahGuaranteedGCInterval); return true; } } @@ -291,3 +301,9 @@ void ShenandoahHeuristics::initialize() { double ShenandoahHeuristics::time_since_last_gc() const { return os::elapsedTime() - _cycle_start; } + +bool ShenandoahHeuristics::in_generation(ShenandoahHeapRegion* region) { + return (_generation->generation_mode() == GLOBAL) + || (_generation->generation_mode() == YOUNG && region->affiliation() == YOUNG_GENERATION) + || (_generation->generation_mode() == OLD && region->affiliation() == OLD_GENERATION); +} diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp index 288d306d08962..52a2bcf3f2bc9 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp @@ -25,7 +25,6 @@ #ifndef SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHHEURISTICS_HPP #define SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHHEURISTICS_HPP -#include "gc/shenandoah/shenandoahHeap.hpp" #include "gc/shenandoah/shenandoahPhaseTimings.hpp" #include "gc/shenandoah/shenandoahSharedVariables.hpp" #include "memory/allocation.hpp" @@ -57,6 +56,7 @@ class ShenandoahCollectionSet; class ShenandoahHeapRegion; +class ShenandoahGeneration; class ShenandoahHeuristics : public CHeapObj { static const intx Concurrent_Adjust = -1; // recover from penalties @@ -84,6 +84,8 @@ class ShenandoahHeuristics : public CHeapObj { // There may be many threads that contend to set this flag ShenandoahSharedFlag _metaspace_oom; + ShenandoahGeneration* _generation; + static int compare_by_garbage(RegionData a, RegionData b); virtual void choose_collection_set_from_regiondata(ShenandoahCollectionSet* set, @@ -92,8 +94,10 @@ class ShenandoahHeuristics : public CHeapObj { void adjust_penalty(intx step); + bool in_generation(ShenandoahHeapRegion* region); + public: - ShenandoahHeuristics(); + ShenandoahHeuristics(ShenandoahGeneration* generation); virtual ~ShenandoahHeuristics(); void record_metaspace_oom() { _metaspace_oom.set(); } diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.hpp index 86ea5651b61bb..a41f2e82cb4fc 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.hpp @@ -29,6 +29,9 @@ class ShenandoahPassiveHeuristics : public ShenandoahHeuristics { public: + ShenandoahPassiveHeuristics(ShenandoahGeneration* generation) + : ShenandoahHeuristics(generation) {} + virtual bool should_start_gc(); virtual bool should_unload_classes(); diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp index db8740d9ae154..978b0a8f91797 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp @@ -27,12 +27,13 @@ #include "gc/shenandoah/heuristics/shenandoahStaticHeuristics.hpp" #include "gc/shenandoah/shenandoahCollectionSet.hpp" #include "gc/shenandoah/shenandoahFreeSet.hpp" -#include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" +#include "gc/shenandoah/shenandoahGeneration.hpp" #include "logging/log.hpp" #include "logging/logTag.hpp" -ShenandoahStaticHeuristics::ShenandoahStaticHeuristics() : ShenandoahHeuristics() { +ShenandoahStaticHeuristics::ShenandoahStaticHeuristics(ShenandoahGeneration* generation) : + ShenandoahHeuristics(generation) { SHENANDOAH_ERGO_ENABLE_FLAG(ExplicitGCInvokesConcurrent); SHENANDOAH_ERGO_ENABLE_FLAG(ShenandoahImplicitGCInvokesConcurrent); } @@ -40,11 +41,9 @@ ShenandoahStaticHeuristics::ShenandoahStaticHeuristics() : ShenandoahHeuristics( ShenandoahStaticHeuristics::~ShenandoahStaticHeuristics() {} bool ShenandoahStaticHeuristics::should_start_gc() { - ShenandoahHeap* heap = ShenandoahHeap::heap(); - - size_t max_capacity = heap->max_capacity(); - size_t capacity = heap->soft_max_capacity(); - size_t available = heap->free_set()->available(); + size_t max_capacity = _generation->max_capacity(); + size_t capacity = _generation->soft_max_capacity(); + size_t available = _generation->available(); // Make sure the code below treats available without the soft tail. size_t soft_tail = max_capacity - capacity; diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.hpp index 5ecd1848d8529..35a41aff8cb93 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.hpp @@ -29,7 +29,7 @@ class ShenandoahStaticHeuristics : public ShenandoahHeuristics { public: - ShenandoahStaticHeuristics(); + ShenandoahStaticHeuristics(ShenandoahGeneration* generation); virtual ~ShenandoahStaticHeuristics(); diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp index 0e9d70d859570..33e2a99616932 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, Red Hat, Inc. All rights reserved. + * Copyright (c) 2019, 2020, Red Hat, Inc. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -23,18 +23,23 @@ */ #include "precompiled.hpp" -#include "gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp" -#include "gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.hpp" -#include "gc/shenandoah/heuristics/shenandoahCompactHeuristics.hpp" -#include "gc/shenandoah/heuristics/shenandoahStaticHeuristics.hpp" #include "gc/shenandoah/mode/shenandoahGenerationalMode.hpp" +#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" #include "logging/log.hpp" #include "logging/logTag.hpp" #include "runtime/globals_extension.hpp" -#include "runtime/java.hpp" void ShenandoahGenerationalMode::initialize_flags() const { + // When we fill in dead objects during update refs, we use oop::size, + // which depends on the klass being loaded. However, if these dead objects + // were the last referrers to the klass, it will be unloaded and we'll + // crash. Class unloading is disabled until we're able to sort this out. + FLAG_SET_ERGO(ClassUnloading, false); + FLAG_SET_ERGO(ClassUnloadingWithConcurrentMark, false); + FLAG_SET_ERGO(ShenandoahUnloadClassesFrequency, 0); + if (ClassUnloading) { + // Leaving this here for the day we re-enable class unloading FLAG_SET_DEFAULT(ShenandoahSuspendibleWorkers, true); FLAG_SET_DEFAULT(VerifyBeforeExit, false); } @@ -48,22 +53,18 @@ void ShenandoahGenerationalMode::initialize_flags() const { SHENANDOAH_CHECK_FLAG_SET(ShenandoahSATBBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahCASBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahCloneBarrier); + SHENANDOAH_CHECK_FLAG_UNSET(ClassUnloading); } -ShenandoahHeuristics* ShenandoahGenerationalMode::initialize_heuristics() const { - if (ShenandoahGCHeuristics != NULL) { - if (strcmp(ShenandoahGCHeuristics, "aggressive") == 0) { - return new ShenandoahAggressiveHeuristics(); - } else if (strcmp(ShenandoahGCHeuristics, "static") == 0) { - return new ShenandoahStaticHeuristics(); - } else if (strcmp(ShenandoahGCHeuristics, "adaptive") == 0) { - return new ShenandoahAdaptiveHeuristics(); - } else if (strcmp(ShenandoahGCHeuristics, "compact") == 0) { - return new ShenandoahCompactHeuristics(); - } else { - vm_exit_during_initialization("Unknown -XX:ShenandoahGCHeuristics option"); - } +const char *affiliation_name(ShenandoahRegionAffiliation type) { + switch (type) { + case ShenandoahRegionAffiliation::FREE: + return "FREE"; + case ShenandoahRegionAffiliation::YOUNG_GENERATION: + return "YOUNG"; + case ShenandoahRegionAffiliation::OLD_GENERATION: + return "OLD"; + default: + return "UnrecognizedAffiliation"; } - ShouldNotReachHere(); - return NULL; } diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp index 6d7482d0bac11..377e7eef40a2d 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp @@ -27,12 +27,23 @@ #include "gc/shenandoah/mode/shenandoahMode.hpp" -class ShenandoahHeuristics; +enum GenerationMode { + YOUNG, + OLD, + GLOBAL +}; + +enum ShenandoahRegionAffiliation { + FREE, + YOUNG_GENERATION, + OLD_GENERATION +}; + +const char *affiliation_name(ShenandoahRegionAffiliation type); class ShenandoahGenerationalMode : public ShenandoahMode { public: virtual void initialize_flags() const; - virtual ShenandoahHeuristics* initialize_heuristics() const; virtual const char* name() { return "Generational"; } virtual bool is_diagnostic() { return false; } virtual bool is_experimental() { return false; } diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahIUMode.cpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahIUMode.cpp index abc4a7cfdc9db..641ef2a567ad4 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahIUMode.cpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahIUMode.cpp @@ -23,10 +23,7 @@ */ #include "precompiled.hpp" -#include "gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp" -#include "gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.hpp" -#include "gc/shenandoah/heuristics/shenandoahCompactHeuristics.hpp" -#include "gc/shenandoah/heuristics/shenandoahStaticHeuristics.hpp" +#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" #include "gc/shenandoah/mode/shenandoahIUMode.hpp" #include "logging/log.hpp" #include "logging/logTag.hpp" @@ -56,21 +53,3 @@ void ShenandoahIUMode::initialize_flags() const { SHENANDOAH_CHECK_FLAG_SET(ShenandoahCASBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahCloneBarrier); } - -ShenandoahHeuristics* ShenandoahIUMode::initialize_heuristics() const { - if (ShenandoahGCHeuristics != NULL) { - if (strcmp(ShenandoahGCHeuristics, "aggressive") == 0) { - return new ShenandoahAggressiveHeuristics(); - } else if (strcmp(ShenandoahGCHeuristics, "static") == 0) { - return new ShenandoahStaticHeuristics(); - } else if (strcmp(ShenandoahGCHeuristics, "adaptive") == 0) { - return new ShenandoahAdaptiveHeuristics(); - } else if (strcmp(ShenandoahGCHeuristics, "compact") == 0) { - return new ShenandoahCompactHeuristics(); - } else { - vm_exit_during_initialization("Unknown -XX:ShenandoahGCHeuristics option"); - } - } - ShouldNotReachHere(); - return NULL; -} diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahIUMode.hpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahIUMode.hpp index c20c483c77daf..455eb4543a845 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahIUMode.hpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahIUMode.hpp @@ -32,8 +32,6 @@ class ShenandoahHeuristics; class ShenandoahIUMode : public ShenandoahMode { public: virtual void initialize_flags() const; - virtual ShenandoahHeuristics* initialize_heuristics() const; - virtual const char* name() { return "Incremental-Update (IU)"; } virtual bool is_diagnostic() { return false; } virtual bool is_experimental() { return true; } diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.cpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.cpp new file mode 100644 index 0000000000000..7ce4b4d0da5d7 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.cpp @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2020, Red Hat, Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp" +#include "gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.hpp" +#include "gc/shenandoah/heuristics/shenandoahCompactHeuristics.hpp" +#include "gc/shenandoah/heuristics/shenandoahStaticHeuristics.hpp" +#include "gc/shenandoah/shenandoahGeneration.hpp" +#include "gc/shenandoah/mode/shenandoahMode.hpp" + +ShenandoahHeuristics* ShenandoahMode::initialize_heuristics(ShenandoahGeneration* generation) const { + + if (ShenandoahGCHeuristics != NULL) { + if (strcmp(ShenandoahGCHeuristics, "aggressive") == 0) { + return new ShenandoahAggressiveHeuristics(generation); + } else if (strcmp(ShenandoahGCHeuristics, "static") == 0) { + return new ShenandoahStaticHeuristics(generation); + } else if (strcmp(ShenandoahGCHeuristics, "adaptive") == 0) { + return new ShenandoahAdaptiveHeuristics(generation); + } else if (strcmp(ShenandoahGCHeuristics, "compact") == 0) { + return new ShenandoahCompactHeuristics(generation); + } else { + vm_exit_during_initialization("Unknown -XX:ShenandoahGCHeuristics option"); + } + } + ShouldNotReachHere(); + return NULL; +} diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.hpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.hpp index 1da6264e19459..6e0975eb7b479 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.hpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.hpp @@ -26,7 +26,10 @@ #define SHARE_GC_SHENANDOAH_MODE_SHENANDOAHMODE_HPP #include "memory/allocation.hpp" +#include "runtime/java.hpp" +#include "utilities/formatBuffer.hpp" +class ShenandoahGeneration; class ShenandoahHeuristics; #define SHENANDOAH_CHECK_FLAG_SET(name) \ @@ -48,7 +51,7 @@ class ShenandoahHeuristics; class ShenandoahMode : public CHeapObj { public: virtual void initialize_flags() const = 0; - virtual ShenandoahHeuristics* initialize_heuristics() const = 0; + virtual ShenandoahHeuristics* initialize_heuristics(ShenandoahGeneration* generation) const; virtual const char* name() = 0; virtual bool is_diagnostic() = 0; virtual bool is_experimental() = 0; diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.cpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.cpp index fb24acdc967a1..6a1834c9d0716 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.cpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.cpp @@ -23,11 +23,11 @@ */ #include "precompiled.hpp" +#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" #include "gc/shenandoah/heuristics/shenandoahPassiveHeuristics.hpp" #include "gc/shenandoah/mode/shenandoahPassiveMode.hpp" #include "logging/log.hpp" #include "logging/logTag.hpp" -#include "runtime/globals_extension.hpp" void ShenandoahPassiveMode::initialize_flags() const { // Do not allow concurrent cycles. @@ -52,9 +52,9 @@ void ShenandoahPassiveMode::initialize_flags() const { // Final configuration checks // No barriers are required to run. } -ShenandoahHeuristics* ShenandoahPassiveMode::initialize_heuristics() const { +ShenandoahHeuristics* ShenandoahPassiveMode::initialize_heuristics(ShenandoahGeneration* generation) const { if (ShenandoahGCHeuristics != NULL) { - return new ShenandoahPassiveHeuristics(); + return new ShenandoahPassiveHeuristics(generation); } ShouldNotReachHere(); return NULL; diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.hpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.hpp index c0e778174b3dd..e39c0cbe8ac50 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.hpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.hpp @@ -30,8 +30,7 @@ class ShenandoahPassiveMode : public ShenandoahMode { public: virtual void initialize_flags() const; - virtual ShenandoahHeuristics* initialize_heuristics() const; - + virtual ShenandoahHeuristics* initialize_heuristics(ShenandoahGeneration* generation) const; virtual const char* name() { return "Passive"; } virtual bool is_diagnostic() { return true; } virtual bool is_experimental() { return false; } diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahSATBMode.cpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahSATBMode.cpp index 3fa15dae146d6..c3f5067d6667a 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahSATBMode.cpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahSATBMode.cpp @@ -23,10 +23,7 @@ */ #include "precompiled.hpp" -#include "gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp" -#include "gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.hpp" -#include "gc/shenandoah/heuristics/shenandoahCompactHeuristics.hpp" -#include "gc/shenandoah/heuristics/shenandoahStaticHeuristics.hpp" +#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" #include "gc/shenandoah/mode/shenandoahSATBMode.hpp" #include "logging/log.hpp" #include "logging/logTag.hpp" @@ -49,21 +46,3 @@ void ShenandoahSATBMode::initialize_flags() const { SHENANDOAH_CHECK_FLAG_SET(ShenandoahCASBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahCloneBarrier); } - -ShenandoahHeuristics* ShenandoahSATBMode::initialize_heuristics() const { - if (ShenandoahGCHeuristics != NULL) { - if (strcmp(ShenandoahGCHeuristics, "aggressive") == 0) { - return new ShenandoahAggressiveHeuristics(); - } else if (strcmp(ShenandoahGCHeuristics, "static") == 0) { - return new ShenandoahStaticHeuristics(); - } else if (strcmp(ShenandoahGCHeuristics, "adaptive") == 0) { - return new ShenandoahAdaptiveHeuristics(); - } else if (strcmp(ShenandoahGCHeuristics, "compact") == 0) { - return new ShenandoahCompactHeuristics(); - } else { - vm_exit_during_initialization("Unknown -XX:ShenandoahGCHeuristics option"); - } - } - ShouldNotReachHere(); - return NULL; -} diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahSATBMode.hpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahSATBMode.hpp index f246f9b20c718..3c2ae5bde93be 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahSATBMode.hpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahSATBMode.hpp @@ -32,7 +32,6 @@ class ShenandoahHeuristics; class ShenandoahSATBMode : public ShenandoahMode { public: virtual void initialize_flags() const; - virtual ShenandoahHeuristics* initialize_heuristics() const; virtual const char* name() { return "Snapshot-At-The-Beginning (SATB)"; } virtual bool is_diagnostic() { return false; } virtual bool is_experimental() { return false; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahAllocRequest.hpp b/src/hotspot/share/gc/shenandoah/shenandoahAllocRequest.hpp index 4f3ec22b6ef47..f32273122f072 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahAllocRequest.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahAllocRequest.hpp @@ -26,12 +26,7 @@ #define SHARE_GC_SHENANDOAH_SHENANDOAHALLOCREQUEST_HPP #include "memory/allocation.hpp" - -enum ShenandoahRegionAffiliation { - FREE, - YOUNG_GENERATION, - OLD_GENERATION -}; +#include "gc/shenandoah/mode/shenandoahGenerationalMode.hpp" class ShenandoahAllocRequest : StackObj { public: @@ -79,11 +74,11 @@ class ShenandoahAllocRequest : StackObj { public: static inline ShenandoahAllocRequest for_tlab(size_t min_size, size_t requested_size) { - return ShenandoahAllocRequest(min_size, requested_size, _alloc_tlab, YOUNG_GENERATION); + return ShenandoahAllocRequest(min_size, requested_size, _alloc_tlab, ShenandoahRegionAffiliation::YOUNG_GENERATION); } static inline ShenandoahAllocRequest for_gclab(size_t min_size, size_t requested_size) { - return ShenandoahAllocRequest(min_size, requested_size, _alloc_gclab, YOUNG_GENERATION); + return ShenandoahAllocRequest(min_size, requested_size, _alloc_gclab, ShenandoahRegionAffiliation::YOUNG_GENERATION); } static inline ShenandoahAllocRequest for_shared_gc(size_t requested_size, ShenandoahRegionAffiliation affiliation) { @@ -91,7 +86,7 @@ class ShenandoahAllocRequest : StackObj { } static inline ShenandoahAllocRequest for_shared(size_t requested_size) { - return ShenandoahAllocRequest(0, requested_size, _alloc_shared, YOUNG_GENERATION); + return ShenandoahAllocRequest(0, requested_size, _alloc_shared, ShenandoahRegionAffiliation::YOUNG_GENERATION); } inline size_t size() { @@ -166,6 +161,14 @@ class ShenandoahAllocRequest : StackObj { } } + bool is_old() { + return _affiliation == OLD_GENERATION; + } + + bool is_young() { + return _affiliation == YOUNG_GENERATION; + } + ShenandoahRegionAffiliation affiliation() const { return _affiliation; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahAsserts.cpp b/src/hotspot/share/gc/shenandoah/shenandoahAsserts.cpp index 27c1f988833ab..2066b594a90e1 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahAsserts.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahAsserts.cpp @@ -71,6 +71,9 @@ void ShenandoahAsserts::print_obj(ShenandoahMessageBuffer& msg, oop obj) { msg.append(" %3s marked strong\n", ctx->is_marked_strong(obj) ? "" : "not"); msg.append(" %3s marked weak\n", ctx->is_marked_weak(obj) ? "" : "not"); msg.append(" %3s in collection set\n", heap->in_collection_set(obj) ? "" : "not"); + if (heap->mode()->is_generational() && !obj->is_forwarded()) { + msg.append(" age: %d\n", obj->age()); + } msg.append(" mark:%s\n", mw_ss.as_string()); msg.append(" region: %s", ss.as_string()); } @@ -363,7 +366,7 @@ void ShenandoahAsserts::assert_locked_or_shenandoah_safepoint(Mutex* lock, const return; } - ShenandoahMessageBuffer msg("Must ba at a Shenandoah safepoint or held %s lock", lock->name()); + ShenandoahMessageBuffer msg("Must be at a Shenandoah safepoint or held %s lock", lock->name()); report_vm_error(file, line, msg.buffer()); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp index ed00d1ed48525..ff7cd374d9c24 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp @@ -137,6 +137,7 @@ void ShenandoahBarrierSet::clone_barrier_runtime(oop src) { clone_barrier(src); } } + void ShenandoahBarrierSet::write_ref_array(HeapWord* start, size_t count) { if (!_heap->mode()->is_generational()) { return; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp index a203b58d7a1db..9f82afff607b3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp @@ -48,7 +48,7 @@ class ShenandoahBarrierSet: public BarrierSet { return barrier_set_cast(BarrierSet::barrier_set()); } - inline CardTable* card_table() { return _card_table; } + inline ShenandoahCardTable* card_table() { return _card_table; } static ShenandoahSATBMarkQueueSet& satb_mark_queue_set() { return barrier_set()->_satb_mark_queue_set; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp index 88210bdda3851..27ac335f35d26 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp @@ -240,7 +240,8 @@ inline oop ShenandoahBarrierSet::AccessBarrier::oop_loa template template inline void ShenandoahBarrierSet::AccessBarrier::oop_store_not_in_heap(T* addr, oop value) { - shenandoah_assert_marked_if(NULL, value, !CompressedOops::is_null(value) && ShenandoahHeap::heap()->is_evacuation_in_progress()); + shenandoah_assert_marked_if(NULL, value, !CompressedOops::is_null(value) && ShenandoahHeap::heap()->is_evacuation_in_progress() && + !(ShenandoahHeap::heap()->is_gc_generation_young() && ShenandoahHeap::heap()->heap_region_containing(value)->is_old())); shenandoah_assert_not_in_cset_if(addr, value, value != NULL && !ShenandoahHeap::heap()->cancelled_gc()); ShenandoahBarrierSet* const bs = ShenandoahBarrierSet::barrier_set(); bs->iu_barrier(value); @@ -352,7 +353,9 @@ bool ShenandoahBarrierSet::AccessBarrier::oop_arraycopy template void ShenandoahBarrierSet::arraycopy_work(T* src, size_t count) { - assert(HAS_FWD == _heap->has_forwarded_objects(), "Forwarded object status is sane"); + // We allow forwarding in young generation and marking in old generation + // to happen simultaneously. + assert(_heap->mode()->is_generational() || HAS_FWD == _heap->has_forwarded_objects(), "Forwarded object status is sane"); Thread* thread = Thread::current(); SATBMarkQueue& queue = ShenandoahThreadLocalData::satb_mark_queue(thread); @@ -372,7 +375,7 @@ void ShenandoahBarrierSet::arraycopy_work(T* src, size_t count) { oop witness = ShenandoahHeap::cas_oop(fwd, elem_ptr, o); obj = fwd; } - if (ENQUEUE && !ctx->is_marked_strong(obj)) { + if (ENQUEUE && !ctx->is_marked_strong_or_old(obj)) { _satb_mark_queue_set.enqueue_known_active(queue, obj); } } @@ -385,13 +388,27 @@ void ShenandoahBarrierSet::arraycopy_barrier(T* src, T* dst, size_t count) { return; } int gc_state = _heap->gc_state(); - if ((gc_state & ShenandoahHeap::MARKING) != 0) { + if ((gc_state & ShenandoahHeap::YOUNG_MARKING) != 0) { arraycopy_marking(src, dst, count); - } else if ((gc_state & ShenandoahHeap::EVACUATION) != 0) { + return; + } + + if ((gc_state & ShenandoahHeap::EVACUATION) != 0) { arraycopy_evacuation(src, count); } else if ((gc_state & ShenandoahHeap::UPDATEREFS) != 0) { arraycopy_update(src, count); } + + if (_heap->mode()->is_generational()) { + assert(ShenandoahSATBBarrier, "Generational mode assumes SATB mode"); + // HEY! Could we optimize here by checking that dst is in an old region? + if ((gc_state & ShenandoahHeap::OLD_MARKING) != 0) { + // Note that we can't do the arraycopy marking using the 'src' array when + // SATB mode is enabled (so we can't do this as part of the iteration for + // evacuation or update references). + arraycopy_marking(src, dst, count); + } + } } template diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp index ca10336feb17a..7322a0e4a7242 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp @@ -102,8 +102,12 @@ void ShenandoahBarrierSet::clone_barrier(oop obj) { assert(ShenandoahCloneBarrier, "only get here with clone barriers enabled"); shenandoah_assert_correct(NULL, obj); + // We only need to handle YOUNG_MARKING here because the clone barrier + // is only invoked during marking if Shenandoah is in incremental update + // mode. OLD_MARKING should only happen when Shenandoah is in generational + // mode, which uses the SATB write barrier. int gc_state = _heap->gc_state(); - if ((gc_state & ShenandoahHeap::MARKING) != 0) { + if ((gc_state & ShenandoahHeap::YOUNG_MARKING) != 0) { clone_marking(obj); } else if ((gc_state & ShenandoahHeap::EVACUATION) != 0) { clone_evacuation(obj); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCardTable.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCardTable.cpp index b9fe2c3292e58..0388175a89a40 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCardTable.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCardTable.cpp @@ -23,9 +23,28 @@ */ #include "precompiled.hpp" +#include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahCardTable.hpp" void ShenandoahCardTable::initialize() { CardTable::initialize(); resize_covered_region(_whole_heap); } + +bool ShenandoahCardTable::is_in_young(oop obj) const { + return ShenandoahHeap::heap()->is_in_young(obj); +} + +bool ShenandoahCardTable::is_dirty(MemRegion mr) { + for (size_t i = index_for(mr.start()); i <= index_for(mr.end() - 1); i++) { + CardValue* byte = byte_for_index(i); + if (*byte == CardTable::dirty_card_val()) { + return true; + } + } + return false; +} + +void ShenandoahCardTable::clear() { + CardTable::clear(_whole_heap); +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCardTable.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCardTable.hpp index a8e2cc32c926d..98e944e662c1d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCardTable.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCardTable.hpp @@ -38,10 +38,11 @@ class ShenandoahCardTable: public CardTable { virtual void initialize(); - inline bool is_in_young(oop obj) const { - ShouldNotReachHere(); - return false; - } + virtual bool is_in_young(oop obj) const; + + bool is_dirty(MemRegion mr); + + void clear(); }; #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHCARDTABLE_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp index 79dd8feaa9535..8f98f9a36e656 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp @@ -45,7 +45,7 @@ bool ShenandoahForwardedIsAliveClosure::do_object_b(oop obj) { } obj = ShenandoahBarrierSet::resolve_forwarded_not_null(obj); shenandoah_assert_not_forwarded_if(NULL, obj, ShenandoahHeap::heap()->is_concurrent_mark_in_progress()); - return _mark_context->is_marked(obj); + return _mark_context->is_marked_or_old(obj); } ShenandoahIsAliveClosure::ShenandoahIsAliveClosure() : @@ -57,7 +57,7 @@ bool ShenandoahIsAliveClosure::do_object_b(oop obj) { return false; } shenandoah_assert_not_forwarded(NULL, obj); - return _mark_context->is_marked(obj); + return _mark_context->is_marked_or_old(obj); } BoolObjectClosure* ShenandoahIsAliveSelector::is_alive_closure() { @@ -81,7 +81,7 @@ void ShenandoahKeepAliveClosure::do_oop(narrowOop* p) { template void ShenandoahKeepAliveClosure::do_oop_work(T* p) { assert(ShenandoahHeap::heap()->is_concurrent_mark_in_progress(), "Only for concurrent marking phase"); - assert(!ShenandoahHeap::heap()->has_forwarded_objects(), "Not expected"); + assert(ShenandoahHeap::heap()->is_concurrent_old_mark_in_progress() || !ShenandoahHeap::heap()->has_forwarded_objects(), "Not expected"); T o = RawAccess<>::oop_load(p); if (!CompressedOops::is_null(o)) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index a6b1102ebaf4e..adfdffb483552 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -47,7 +47,7 @@ #include "utilities/events.hpp" ShenandoahConcurrentGC::ShenandoahConcurrentGC(ShenandoahGeneration* generation) : - _mark(), + _mark(generation), _degen_point(ShenandoahDegenPoint::_degenerated_unset), _generation(generation) { } @@ -56,10 +56,6 @@ ShenandoahGC::ShenandoahDegenPoint ShenandoahConcurrentGC::degen_point() const { return _degen_point; } -void ShenandoahConcurrentGC::cancel() { - ShenandoahConcurrentMark::cancel(); -} - bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) { ShenandoahHeap* const heap = ShenandoahHeap::heap(); @@ -186,7 +182,8 @@ void ShenandoahConcurrentGC::vmop_entry_final_updaterefs() { } void ShenandoahConcurrentGC::entry_init_mark() { - const char* msg = init_mark_event_message(); + char msg[1024]; + init_mark_event_message(msg, sizeof(msg)); ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_mark); EventMark em("%s", msg); @@ -198,7 +195,8 @@ void ShenandoahConcurrentGC::entry_init_mark() { } void ShenandoahConcurrentGC::entry_final_mark() { - const char* msg = final_mark_event_message(); + char msg[1024]; + final_mark_event_message(msg, sizeof(msg)); ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_mark); EventMark em("%s", msg); @@ -261,9 +259,10 @@ void ShenandoahConcurrentGC::entry_mark_roots() { } void ShenandoahConcurrentGC::entry_mark() { + char msg[1024]; ShenandoahHeap* const heap = ShenandoahHeap::heap(); TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); - const char* msg = conc_mark_event_message(); + conc_mark_event_message(msg, sizeof(msg)); ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_mark); EventMark em("%s", msg); @@ -436,7 +435,7 @@ void ShenandoahConcurrentGC::op_reset() { heap->pacer()->setup_for_reset(); } - heap->prepare_gc(); + _generation->prepare_gc(); } class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure { @@ -467,8 +466,8 @@ void ShenandoahConcurrentGC::op_init_mark() { assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint"); assert(Thread::current()->is_VM_thread(), "can only do this in VMThread"); - assert(heap->marking_context()->is_bitmap_clear(), "need clear marking bitmap"); - assert(!heap->marking_context()->is_complete(), "should not be complete"); + assert(_generation->is_bitmap_clear(), "need clear marking bitmap"); + assert(!_generation->is_mark_complete(), "should not be complete"); assert(!heap->has_forwarded_objects(), "No forwarded objects on this path"); if (ShenandoahVerify) { @@ -479,12 +478,12 @@ void ShenandoahConcurrentGC::op_init_mark() { Universe::verify(); } - heap->set_concurrent_mark_in_progress(true); + _generation->set_concurrent_mark_in_progress(true); { ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states); ShenandoahInitMarkUpdateRegionStateClosure cl; - heap->parallel_heap_region_iterate(&cl); + _generation->parallel_heap_region_iterate(&cl); } // Weak reference processing @@ -494,6 +493,11 @@ void ShenandoahConcurrentGC::op_init_mark() { // Make above changes visible to worker threads OrderAccess::fence(); + + if (_generation->generation_mode() == YOUNG) { + _generation->scan_remembered_set(); + } + // Arm nmethods for concurrent marking. When a nmethod is about to be executed, // we need to make sure that all its metadata are marked. alternative is to remark // thread roots at final mark pause, but it can be potential latency killer. @@ -508,11 +512,11 @@ void ShenandoahConcurrentGC::op_init_mark() { } void ShenandoahConcurrentGC::op_mark_roots() { - _mark.mark_concurrent_roots(_generation); + _mark.mark_concurrent_roots(); } void ShenandoahConcurrentGC::op_mark() { - _mark.concurrent_mark(_generation); + _mark.concurrent_mark(); } void ShenandoahConcurrentGC::op_final_mark() { @@ -525,13 +529,13 @@ void ShenandoahConcurrentGC::op_final_mark() { } if (!heap->cancelled_gc()) { - _mark.finish_mark(_generation); + _mark.finish_mark(); assert(!heap->cancelled_gc(), "STW mark cannot OOM"); // Notify JVMTI that the tagmap table will need cleaning. JvmtiTagMap::set_needs_cleaning(); - heap->prepare_regions_and_collection_set(true /*concurrent*/); + _generation->prepare_regions_and_collection_set(true /*concurrent*/); // Has to be done after cset selection heap->prepare_concurrent_roots(); @@ -939,7 +943,7 @@ void ShenandoahConcurrentGC::op_final_updaterefs() { // Clear cancelled GC, if set. On cancellation path, the block before would handle // everything. if (heap->cancelled_gc()) { - heap->clear_cancelled_gc(); + heap->clear_cancelled_gc(true /* clear oom handler */); } // Has to be done before cset is clear @@ -949,6 +953,13 @@ void ShenandoahConcurrentGC::op_final_updaterefs() { heap->update_heap_region_states(true /*concurrent*/); + if (heap->is_concurrent_old_mark_in_progress()) { + // Purge the SATB buffers, transferring any valid, old pointers to the + // old generation mark queue. From here on, no mutator will have access + // to anything that will be trashed and recycled. + heap->purge_old_satb_buffers(false /* abandon */); + } + heap->set_update_refs_in_progress(false); heap->set_has_forwarded_objects(false); @@ -975,32 +986,34 @@ bool ShenandoahConcurrentGC::check_cancellation_and_abort(ShenandoahDegenPoint p return false; } -const char* ShenandoahConcurrentGC::init_mark_event_message() const { +void ShenandoahConcurrentGC::init_mark_event_message(char* buf, size_t len) const { ShenandoahHeap* const heap = ShenandoahHeap::heap(); assert(!heap->has_forwarded_objects(), "Should not have forwarded objects here"); if (heap->unload_classes()) { - return "Pause Init Mark (unload classes)"; + jio_snprintf(buf, len, "Pause Init Mark (%s) (unload classes)", _generation->name()); } else { - return "Pause Init Mark"; + jio_snprintf(buf, len, "Pause Init Mark (%s)", _generation->name()); } } -const char* ShenandoahConcurrentGC::final_mark_event_message() const { +void ShenandoahConcurrentGC::final_mark_event_message(char* buf, size_t len) const { ShenandoahHeap* const heap = ShenandoahHeap::heap(); - assert(!heap->has_forwarded_objects(), "Should not have forwarded objects here"); + assert(!heap->has_forwarded_objects() || heap->is_concurrent_old_mark_in_progress(), + "Should not have forwarded objects during final mark (unless old gen concurrent mark is running)"); if (heap->unload_classes()) { - return "Pause Final Mark (unload classes)"; + jio_snprintf(buf, len, "Pause Final Mark (%s) (unload classes)", _generation->name()); } else { - return "Pause Final Mark"; + jio_snprintf(buf, len, "Pause Final Mark (%s)", _generation->name()); } } -const char* ShenandoahConcurrentGC::conc_mark_event_message() const { +void ShenandoahConcurrentGC::conc_mark_event_message(char* buf, size_t len) const { ShenandoahHeap* const heap = ShenandoahHeap::heap(); - assert(!heap->has_forwarded_objects(), "Should not have forwarded objects here"); + assert(!heap->has_forwarded_objects() || heap->is_concurrent_old_mark_in_progress(), + "Should not have forwarded objects concurrent mark (unless old gen concurrent mark is running"); if (heap->unload_classes()) { - return "Concurrent marking (unload classes)"; + jio_snprintf(buf, len, "Concurrent marking (%s) (unload classes)", _generation->name()); } else { - return "Concurrent marking"; + jio_snprintf(buf, len, "Concurrent marking (%s)", _generation->name()); } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp index c67b0ac7df6f5..6dee549671a2b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp @@ -53,13 +53,15 @@ class ShenandoahConcurrentGC : public ShenandoahGC { bool collect(GCCause::Cause cause); ShenandoahDegenPoint degen_point() const; - // Cancel ongoing concurrent GC - static void cancel(); private: // Entry points to STW GC operations, these cause a related safepoint, that then // call the entry method below void vmop_entry_init_mark(); + +protected: void vmop_entry_final_mark(); + +private: void vmop_entry_init_updaterefs(); void vmop_entry_final_updaterefs(); @@ -74,6 +76,8 @@ class ShenandoahConcurrentGC : public ShenandoahGC { // for concurrent operation. void entry_reset(); void entry_mark_roots(); + +protected: void entry_mark(); void entry_thread_roots(); void entry_weak_refs(); @@ -82,6 +86,8 @@ class ShenandoahConcurrentGC : public ShenandoahGC { void entry_strong_roots(); void entry_cleanup_early(); void entry_rendezvous_roots(); + +private: void entry_evacuate(); void entry_update_thread_roots(); void entry_updaterefs(); @@ -109,10 +115,11 @@ class ShenandoahConcurrentGC : public ShenandoahGC { // Messages for GC trace events, they have to be immortal for // passing around the logging/tracing systems - const char* init_mark_event_message() const; - const char* final_mark_event_message() const; - const char* conc_mark_event_message() const; + void init_mark_event_message(char* buf, size_t len) const; + void final_mark_event_message(char* buf, size_t len) const; + void conc_mark_event_message(char* buf, size_t len) const; +protected: // Check GC cancellation and abort concurrent GC bool check_cancellation_and_abort(ShenandoahDegenPoint point); }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp index 6d40e5fcaec09..525126bfb3e53 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp @@ -86,7 +86,6 @@ class ShenandoahConcurrentMarkingTask : public AbstractGangTask { ShenandoahHeap* heap = ShenandoahHeap::heap(); ShenandoahConcurrentWorkerSession worker_session(worker_id); ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers); - ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id); ShenandoahReferenceProcessor* rp = heap->ref_processor(); assert(rp != NULL, "need reference processor"); _cm->mark_loop(GENERATION, worker_id, _terminator, rp, @@ -142,13 +141,14 @@ class ShenandoahFinalMarkingTask : public AbstractGangTask { // First drain remaining SATB buffers. { ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id); + ShenandoahObjToScanQueue* old = _cm->get_old_queue(worker_id); - ShenandoahSATBBufferClosure cl(q); + ShenandoahSATBBufferClosure cl(q, old); SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set(); while (satb_mq_set.apply_closure_to_completed_buffer(&cl)) {} assert(!heap->has_forwarded_objects(), "Not expected"); - ShenandoahMarkRefsClosure mark_cl(q, rp); + ShenandoahMarkRefsClosure mark_cl(q, rp, old); ShenandoahSATBAndRemarkThreadsClosure tc(satb_mq_set, ShenandoahIUBarrier ? &mark_cl : NULL); Threads::threads_do(&tc); @@ -162,22 +162,22 @@ class ShenandoahFinalMarkingTask : public AbstractGangTask { } }; -ShenandoahConcurrentMark::ShenandoahConcurrentMark() : - ShenandoahMark() {} +ShenandoahConcurrentMark::ShenandoahConcurrentMark(ShenandoahGeneration* generation) : + ShenandoahMark(generation) {} // Mark concurrent roots during concurrent phases template class ShenandoahMarkConcurrentRootsTask : public AbstractGangTask { private: - ShenandoahConcurrentMark* _scm; SuspendibleThreadSetJoiner _sts_joiner; ShenandoahConcurrentRootScanner _root_scanner; ShenandoahObjToScanQueueSet* const _queue_set; + ShenandoahObjToScanQueueSet* const _old_queue_set; ShenandoahReferenceProcessor* const _rp; public: - ShenandoahMarkConcurrentRootsTask(ShenandoahConcurrentMark* scm, - ShenandoahObjToScanQueueSet* qs, + ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs, + ShenandoahObjToScanQueueSet* old, ShenandoahReferenceProcessor* rp, ShenandoahPhaseTimings::Phase phase, uint nworkers); @@ -185,14 +185,15 @@ class ShenandoahMarkConcurrentRootsTask : public AbstractGangTask { }; template -ShenandoahMarkConcurrentRootsTask::ShenandoahMarkConcurrentRootsTask(ShenandoahConcurrentMark* scm, - ShenandoahObjToScanQueueSet* qs, +ShenandoahMarkConcurrentRootsTask::ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs, + ShenandoahObjToScanQueueSet* old, ShenandoahReferenceProcessor* rp, ShenandoahPhaseTimings::Phase phase, uint nworkers) : AbstractGangTask("Shenandoah Concurrent Mark Roots"), _root_scanner(nworkers, phase), _queue_set(qs), + _old_queue_set(old), _rp(rp) { assert(!ShenandoahHeap::heap()->has_forwarded_objects(), "Not expected"); } @@ -201,11 +202,12 @@ template void ShenandoahMarkConcurrentRootsTask::work(uint worker_id) { ShenandoahConcurrentWorkerSession worker_session(worker_id); ShenandoahObjToScanQueue* q = _queue_set->queue(worker_id); - ShenandoahMarkRefsClosure cl(q, _rp); + ShenandoahObjToScanQueue* old = _old_queue_set == NULL ? NULL : _old_queue_set->queue(worker_id); + ShenandoahMarkRefsClosure cl(q, _rp, old); _root_scanner.roots_do(&cl, worker_id); } -void ShenandoahConcurrentMark::mark_concurrent_roots(ShenandoahGeneration* generation) { +void ShenandoahConcurrentMark::mark_concurrent_roots() { ShenandoahHeap* const heap = ShenandoahHeap::heap(); assert(!heap->has_forwarded_objects(), "Not expected"); @@ -213,19 +215,22 @@ void ShenandoahConcurrentMark::mark_concurrent_roots(ShenandoahGeneration* gener WorkGang* workers = heap->workers(); ShenandoahReferenceProcessor* rp = heap->ref_processor(); - task_queues()->reserve(workers->active_workers()); - switch (generation->generation_mode()) { + _generation->reserve_task_queues(workers->active_workers()); + switch (_generation->generation_mode()) { case YOUNG: { - ShenandoahMarkConcurrentRootsTask task(this, task_queues(), rp, ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers()); + ShenandoahMarkConcurrentRootsTask task(task_queues(), old_task_queues(), rp, ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers()); workers->run_task(&task); break; } case GLOBAL: { - ShenandoahMarkConcurrentRootsTask task(this, task_queues(), rp, ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers()); + assert(old_task_queues() == NULL, "Global mark should not have old gen mark queues."); + ShenandoahMarkConcurrentRootsTask task(task_queues(), old_task_queues(), rp, ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers()); workers->run_task(&task); break; } default: + // Intentionally haven't added OLD here. We use a YOUNG generation + // cycle to bootstrap concurrent old marking. ShouldNotReachHere(); } } @@ -243,7 +248,7 @@ class ShenandoahFlushSATBHandshakeClosure : public HandshakeClosure { } }; -void ShenandoahConcurrentMark::concurrent_mark(ShenandoahGeneration* generation) { +void ShenandoahConcurrentMark::concurrent_mark() { ShenandoahHeap* const heap = ShenandoahHeap::heap(); WorkGang* workers = heap->workers(); uint nworkers = workers->active_workers(); @@ -252,13 +257,19 @@ void ShenandoahConcurrentMark::concurrent_mark(ShenandoahGeneration* generation) ShenandoahSATBMarkQueueSet& qset = ShenandoahBarrierSet::satb_mark_queue_set(); ShenandoahFlushSATBHandshakeClosure flush_satb(qset); for (uint flushes = 0; flushes < ShenandoahMaxSATBBufferFlushes; flushes++) { - switch (generation->generation_mode()) { + switch (_generation->generation_mode()) { case YOUNG: { TaskTerminator terminator(nworkers, task_queues()); ShenandoahConcurrentMarkingTask task(this, &terminator); workers->run_task(&task); break; } + case OLD: { + TaskTerminator terminator(nworkers, task_queues()); + ShenandoahConcurrentMarkingTask task(this, &terminator); + workers->run_task(&task); + break; + } case GLOBAL: { TaskTerminator terminator(nworkers, task_queues()); ShenandoahConcurrentMarkingTask task(this, &terminator); @@ -286,20 +297,19 @@ void ShenandoahConcurrentMark::concurrent_mark(ShenandoahGeneration* generation) assert(task_queues()->is_empty() || heap->cancelled_gc(), "Should be empty when not cancelled"); } -void ShenandoahConcurrentMark::finish_mark(ShenandoahGeneration* generation) { +void ShenandoahConcurrentMark::finish_mark() { assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint"); assert(Thread::current()->is_VM_thread(), "Must by VM Thread"); - finish_mark_work(generation); + finish_mark_work(); assert(task_queues()->is_empty(), "Should be empty"); TASKQUEUE_STATS_ONLY(task_queues()->print_taskqueue_stats()); TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats()); - ShenandoahHeap* const heap = ShenandoahHeap::heap(); - heap->set_concurrent_mark_in_progress(false); - heap->mark_complete_marking_context(); + _generation->set_concurrent_mark_in_progress(false); + _generation->set_mark_complete(); } -void ShenandoahConcurrentMark::finish_mark_work(ShenandoahGeneration* generation) { +void ShenandoahConcurrentMark::finish_mark_work() { // Finally mark everything else we've got in our queues during the previous steps. // It does two different things for concurrent vs. mark-compact GC: // - For concurrent GC, it starts with empty task queues, drains the remaining @@ -315,12 +325,17 @@ void ShenandoahConcurrentMark::finish_mark_work(ShenandoahGeneration* generation StrongRootsScope scope(nworkers); TaskTerminator terminator(nworkers, task_queues()); - switch (generation->generation_mode()) { + switch (_generation->generation_mode()) { case YOUNG:{ ShenandoahFinalMarkingTask task(this, &terminator, ShenandoahStringDedup::is_enabled()); heap->workers()->run_task(&task); break; } + case OLD:{ + ShenandoahFinalMarkingTask task(this, &terminator, ShenandoahStringDedup::is_enabled()); + heap->workers()->run_task(&task); + break; + } case GLOBAL:{ ShenandoahFinalMarkingTask task(this, &terminator, ShenandoahStringDedup::is_enabled()); heap->workers()->run_task(&task); @@ -333,10 +348,3 @@ void ShenandoahConcurrentMark::finish_mark_work(ShenandoahGeneration* generation assert(task_queues()->is_empty(), "Should be empty"); } - - -void ShenandoahConcurrentMark::cancel() { - clear(); - ShenandoahReferenceProcessor* rp = ShenandoahHeap::heap()->ref_processor(); - rp->abandon_partial_discovery(); -} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp index 0fcd6aece8b0e..4741a05b4c93b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp @@ -38,20 +38,19 @@ class ShenandoahConcurrentMark: public ShenandoahMark { template friend class ShenandoahFinalMarkingTask; public: - ShenandoahConcurrentMark(); + ShenandoahConcurrentMark(ShenandoahGeneration* generation); // Concurrent mark roots - void mark_concurrent_roots(ShenandoahGeneration* generation); + void mark_concurrent_roots(); // Concurrent mark - void concurrent_mark(ShenandoahGeneration* generation); - // Finish mark at a safepoint - void finish_mark(ShenandoahGeneration* generation); + void concurrent_mark(); - static void cancel(); + // Finish mark at a safepoint + void finish_mark(); private: - void finish_mark_work(ShenandoahGeneration* generation); + void finish_mark_work(); }; #endif // SHARE_GC_SHENANDOAH_SHENANDOAHCONCURRENTMARK_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index b1f03383016e0..54de48d8c9a5b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -31,6 +31,7 @@ #include "gc/shenandoah/shenandoahFreeSet.hpp" #include "gc/shenandoah/shenandoahFullGC.hpp" #include "gc/shenandoah/shenandoahGeneration.hpp" +#include "gc/shenandoah/shenandoahYoungGeneration.hpp" #include "gc/shenandoah/shenandoahPhaseTimings.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahMark.inline.hpp" @@ -45,6 +46,7 @@ #include "memory/iterator.hpp" #include "memory/universe.hpp" #include "runtime/atomic.hpp" +#include "shenandoahOldGC.hpp" ShenandoahControlThread::ShenandoahControlThread() : ConcurrentGCThread(), @@ -52,8 +54,11 @@ ShenandoahControlThread::ShenandoahControlThread() : _gc_waiters_lock(Mutex::leaf, "ShenandoahRequestedGC_lock", true, Monitor::_safepoint_check_always), _periodic_task(this), _requested_gc_cause(GCCause::_no_cause_specified), + _requested_generation(GenerationMode::GLOBAL), _degen_point(ShenandoahGC::_degenerated_outside_cycle), - _allocs_seen(0) { + _degen_generation(NULL), + _allocs_seen(0), + _mode(none) { reset_gc_id(); create_and_start(); @@ -81,11 +86,10 @@ void ShenandoahControlThread::run_service() { ShenandoahHeap* heap = ShenandoahHeap::heap(); GCMode default_mode = concurrent_normal; + GenerationMode generation = GLOBAL; GCCause::Cause default_cause = GCCause::_shenandoah_concurrent_gc; - int sleep = ShenandoahControlIntervalMin; double last_shrink_time = os::elapsedTime(); - double last_sleep_adjust_time = os::elapsedTime(); // Shrink period avoids constantly polling regions for shrinking. // Having a period 10x lower than the delay would mean we hit the @@ -94,12 +98,18 @@ void ShenandoahControlThread::run_service() { double shrink_period = (double)ShenandoahUncommitDelay / 1000 / 10; ShenandoahCollectorPolicy* policy = heap->shenandoah_policy(); - ShenandoahHeuristics* heuristics = heap->heuristics(); + + // HEY! heuristics are notified of allocation failures here and other outcomes + // of the cycle. They're also used here to control whether the Nth consecutive + // degenerated cycle should be 'promoted' to a full cycle. This changes the + // threading model for them somewhat, as they are now evaluated on a separate + // thread. + ShenandoahHeuristics* global_heuristics = heap->global_generation()->heuristics(); while (!in_graceful_shutdown() && !should_terminate()) { // Figure out if we have pending requests. bool alloc_failure_pending = _alloc_failure_gc.is_set(); - bool explicit_gc_requested = _gc_requested.is_set() && is_explicit_gc(_requested_gc_cause); - bool implicit_gc_requested = _gc_requested.is_set() && !is_explicit_gc(_requested_gc_cause); + bool explicit_gc_requested = _gc_requested.is_set() && is_explicit_gc(_requested_gc_cause); + bool implicit_gc_requested = _gc_requested.is_set() && is_implicit_gc(_requested_gc_cause); // This control loop iteration have seen this much allocations. size_t allocs_seen = Atomic::xchg(&_allocs_seen, (size_t)0, memory_order_relaxed); @@ -108,7 +118,7 @@ void ShenandoahControlThread::run_service() { bool soft_max_changed = check_soft_max_changed(); // Choose which GC mode to run in. The block below should select a single mode. - GCMode mode = none; + set_gc_mode(none); GCCause::Cause cause = GCCause::_last_gc_cause; ShenandoahGC::ShenandoahDegenPoint degen_point = ShenandoahGC::_degenerated_unset; @@ -122,70 +132,101 @@ void ShenandoahControlThread::run_service() { degen_point = _degen_point; _degen_point = ShenandoahGC::_degenerated_outside_cycle; + if (degen_point == ShenandoahGC::_degenerated_outside_cycle) { + _degen_generation = heap->global_generation(); + } else { + assert(_degen_generation != NULL, "Need to know which generation to resume."); + } + + ShenandoahHeuristics* heuristics = _degen_generation->heuristics(); + generation = _degen_generation->generation_mode(); + if (ShenandoahDegeneratedGC && heuristics->should_degenerate_cycle()) { heuristics->record_allocation_failure_gc(); policy->record_alloc_failure_to_degenerated(degen_point); - mode = stw_degenerated; + set_gc_mode(stw_degenerated); } else { heuristics->record_allocation_failure_gc(); policy->record_alloc_failure_to_full(); - mode = stw_full; + set_gc_mode(stw_full); } - } else if (explicit_gc_requested) { + generation = GLOBAL; cause = _requested_gc_cause; log_info(gc)("Trigger: Explicit GC request (%s)", GCCause::to_string(cause)); - heuristics->record_requested_gc(); + global_heuristics->record_requested_gc(); if (ExplicitGCInvokesConcurrent) { policy->record_explicit_to_concurrent(); - mode = default_mode; + set_gc_mode(default_mode); // Unload and clean up everything - heap->set_unload_classes(heuristics->can_unload_classes()); + heap->set_unload_classes(global_heuristics->can_unload_classes()); } else { policy->record_explicit_to_full(); - mode = stw_full; + set_gc_mode(stw_full); } } else if (implicit_gc_requested) { + generation = GLOBAL; cause = _requested_gc_cause; log_info(gc)("Trigger: Implicit GC request (%s)", GCCause::to_string(cause)); - heuristics->record_requested_gc(); + global_heuristics->record_requested_gc(); if (ShenandoahImplicitGCInvokesConcurrent) { policy->record_implicit_to_concurrent(); - mode = default_mode; + set_gc_mode(default_mode); // Unload and clean up everything - heap->set_unload_classes(heuristics->can_unload_classes()); + heap->set_unload_classes(global_heuristics->can_unload_classes()); } else { policy->record_implicit_to_full(); - mode = stw_full; + set_gc_mode(stw_full); } } else { - // Potential normal cycle: ask heuristics if it wants to act - if (heuristics->should_start_gc()) { - mode = default_mode; - cause = default_cause; + // We should only be here if the regulator requested a cycle or if + // there is an old generation mark in progress. + if (_requested_gc_cause == GCCause::_shenandoah_concurrent_gc) { + // preemption was requested or this is a regular cycle + cause = GCCause::_shenandoah_concurrent_gc; + generation = _requested_generation; + set_gc_mode(default_mode); + + // Don't start a new old marking if there is one already in progress. + if (generation == OLD && heap->is_concurrent_old_mark_in_progress()) { + set_gc_mode(marking_old); + } + + if (generation == GLOBAL) { + heap->set_unload_classes(global_heuristics->should_unload_classes()); + } else { + heap->set_unload_classes(false); + } + } else if (heap->is_concurrent_old_mark_in_progress()) { + // Nobody asked us to do anything, but we have an old generation mark + // in progress, so resume working on that. + cause = GCCause::_shenandoah_concurrent_gc; + generation = OLD; + set_gc_mode(marking_old); } - // Ask policy if this cycle wants to process references or unload classes - heap->set_unload_classes(heuristics->should_unload_classes()); + // Don't want to spin in this loop and start a cycle every time, so + // clear requested gc cause. This creates a race with callers of the + // blocking 'request_gc' method, but there it loops and resets the + // '_requested_gc_cause' until a full cycle is completed. + _requested_gc_cause = GCCause::_no_gc; } // Blow all soft references on this cycle, if handling allocation failure, - // either implicit or explicit GC request, or we are requested to do so unconditionally. - if (alloc_failure_pending || implicit_gc_requested || explicit_gc_requested || ShenandoahAlwaysClearSoftRefs) { + // either implicit or explicit GC request, or we are requested to do so unconditionally. + if (generation == GLOBAL && (alloc_failure_pending || implicit_gc_requested || explicit_gc_requested || ShenandoahAlwaysClearSoftRefs)) { heap->soft_ref_policy()->set_should_clear_all_soft_refs(true); } - bool gc_requested = (mode != none); + bool gc_requested = (_mode != none); assert (!gc_requested || cause != GCCause::_last_gc_cause, "GC cause should be set"); if (gc_requested) { - // GC is starting, bump the internal ID - update_gc_id(); heap->reset_bytes_allocated_since_gc_start(); @@ -202,26 +243,34 @@ void ShenandoahControlThread::run_service() { heap->free_set()->log_status(); } - switch (mode) { - case concurrent_normal: - if (heap->mode()->is_generational()) { - // TODO: Only young collections for now. - // We'll add old collections later. - service_concurrent_normal_cycle(cause, heap->young_generation()); - } else { - service_concurrent_normal_cycle(cause, heap->global_generation()); + { + switch (_mode) { + case concurrent_normal: { + service_concurrent_normal_cycle(heap, generation, cause); + break; + } + case stw_degenerated: { + service_stw_degenerated_cycle(cause, degen_point); + break; } - break; - case stw_degenerated: - service_stw_degenerated_cycle(cause, degen_point); - break; - case stw_full: - service_stw_full_cycle(cause); - break; - default: - ShouldNotReachHere(); + case stw_full: { + service_stw_full_cycle(cause); + break; + } + case marking_old: { + assert(generation == OLD, "Expected old generation here"); + resume_concurrent_old_cycle(heap->old_generation(), cause); + break; + } + default: { + ShouldNotReachHere(); + } + } } + // GC is finished, bump the internal ID before notifying waiters. + update_gc_id(); + // If this was the requested GC cycle, notify waiters about it if (explicit_gc_requested || implicit_gc_requested) { notify_gc_waiters(); @@ -257,7 +306,9 @@ void ShenandoahControlThread::run_service() { // Clear metaspace oom flag, if current cycle unloaded classes if (heap->unload_classes()) { - heuristics->clear_metaspace_oom(); + // HEY! Should we do this if the cycle was cancelled/degenerated? + assert(generation != YOUNG, "should not unload classes in young cycle"); + global_heuristics->clear_metaspace_oom(); } // Commit worker statistics to cycle data @@ -316,16 +367,11 @@ void ShenandoahControlThread::run_service() { last_shrink_time = current; } - // Wait before performing the next action. If allocation happened during this wait, - // we exit sooner, to let heuristics re-evaluate new conditions. If we are at idle, - // back off exponentially. - if (_heap_changed.try_unset()) { - sleep = ShenandoahControlIntervalMin; - } else if ((current - last_sleep_adjust_time) * 1000 > ShenandoahControlIntervalAdjustPeriod){ - sleep = MIN2(ShenandoahControlIntervalMax, MAX2(1, sleep * 2)); - last_sleep_adjust_time = current; - } - os::naked_short_sleep(sleep); + // HEY! kemperw would like to have this thread sleep on a timed wait so it + // could be explicitly woken when there is something to do. The timed wait + // is necessary because this thread has a responsibility to send + // 'alloc_words' to the pacer when it does not perform a GC. + os::naked_short_sleep(ShenandoahControlIntervalMin); } // Wait for the actual stop(), can't leave run_service() earlier. @@ -334,6 +380,107 @@ void ShenandoahControlThread::run_service() { } } +// Young and old concurrent cycles are initiated by the regulator. Implicit +// and explicit GC requests are handled by the controller thread and always +// run a global cycle (which is concurrent by default, but may be overridden +// by command line options). Old cycles always degenerate to a global cycle. +// Young cycles are degenerated to complete the young cycle. +// +// +// +-------------+----------+ Idle +--------------+ +// | | + | +// | | | | +// | | | | +// | | | | +// | v v v +// | +// | Young <---+ Resume Old <----+ Bootstrap Old +// | + + + + +// | | | | | +// | v | | v +// v Young Degen | | Young Degen +// | | +// Global <--------------------+ | +// + | +// | v +// | +// +---------------------> Global Degen +// +void ShenandoahControlThread::service_concurrent_normal_cycle( + const ShenandoahHeap* heap, const GenerationMode generation, GCCause::Cause cause) { + + switch (generation) { + case YOUNG: { + // Run a young cycle. This might or might not, have interrupted an ongoing + // concurrent mark in the old generation. We need to think about promotions + // in this case. Promoted objects should be above the TAMS in the old regions + // they end up in, but we have to be sure we don't promote into any regions + // that are in the cset (more of an issue for Milestone-8 to worry about). + log_info(gc, ergo)("Start GC cycle (YOUNG)"); + service_concurrent_cycle(heap->young_generation(), cause); + heap->young_generation()->log_status(); + break; + } + case GLOBAL: { + log_info(gc, ergo)("Start GC cycle (GLOBAL)"); + service_concurrent_cycle(heap->global_generation(), cause); + heap->global_generation()->log_status(); + break; + } + case OLD: { + log_info(gc, ergo)("Start GC cycle (OLD)"); + service_concurrent_old_cycle(heap, cause); + heap->old_generation()->log_status(); + break; + } + default: + ShouldNotReachHere(); + } +} + +void ShenandoahControlThread::service_concurrent_old_cycle(const ShenandoahHeap* heap, GCCause::Cause &cause) { + // Configure the young generation's concurrent mark to put objects in + // old regions into the concurrent mark queues associated with the old + // generation. The young cycle will run as normal except that rather than + // ignore old references it will mark and enqueue them in the old concurrent + // mark but it will not traverse them. + ShenandoahGeneration* old_generation = heap->old_generation(); + ShenandoahYoungGeneration* young_generation = heap->young_generation(); + + assert(!heap->is_concurrent_old_mark_in_progress(), "Old already in progress."); + assert(old_generation->task_queues()->is_empty(), "Old mark queues should be empty."); + + young_generation->set_old_gen_task_queues(old_generation->task_queues()); + + old_generation->set_mark_incomplete(); + + service_concurrent_cycle(young_generation, cause); + + // Young generation no longer needs this reference to the old concurrent + // mark so clean it up. + young_generation->set_old_gen_task_queues(NULL); + + if (!heap->cancelled_gc()) { + // Reset the degenerated point. Normally this would happen at the top + // of the control loop, but here we have just completed a young cycle + // which has bootstrapped the old concurrent marking. + _degen_point = ShenandoahGC::_degenerated_outside_cycle; + + // TODO: Bit of a hack here to keep the phase timings happy as we transition + // to concurrent old marking. We need to revisit this. + heap->phase_timings()->flush_par_workers_to_cycle(); + heap->phase_timings()->flush_cycle_to_global(); + + // From here we will 'resume' the old concurrent mark. This will skip reset + // and init mark for the concurrent mark. All of that work will have been + // done by the bootstrapping young cycle. In order to simplify the debugging + // effort, the old cycle will ONLY complete the mark phase. No actual + // collection of the old generation is happening here. + set_gc_mode(marking_old); + resume_concurrent_old_cycle(old_generation, cause); + } +} + bool ShenandoahControlThread::check_soft_max_changed() const { ShenandoahHeap* heap = ShenandoahHeap::heap(); size_t new_soft_max = Atomic::load(&SoftMaxHeapSize); @@ -353,7 +500,42 @@ bool ShenandoahControlThread::check_soft_max_changed() const { return false; } -void ShenandoahControlThread::service_concurrent_normal_cycle(GCCause::Cause cause, ShenandoahGeneration* generation) { +void ShenandoahControlThread::resume_concurrent_old_cycle(ShenandoahGeneration* generation, GCCause::Cause cause) { + assert(ShenandoahHeap::heap()->is_concurrent_old_mark_in_progress(), "Old mark should be in progress"); + log_debug(gc)("Resuming old generation with " UINT32_FORMAT " marking tasks queued.", generation->task_queues()->tasks()); + + ShenandoahHeap* heap = ShenandoahHeap::heap(); + + GCIdMark gc_id_mark; + ShenandoahGCSession session(cause, generation); + + TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); + + // We can only really tolerate being cancelled during concurrent + // marking. This flag here (passed by reference) is used to control + // precisely where the regulator is allowed to cancel a GC. + ShenandoahOldGC gc(generation, _allow_old_preemption); + if (gc.collect(cause)) { + // Cycle is complete + generation->heuristics()->record_success_concurrent(); + heap->shenandoah_policy()->record_success_concurrent(); + } + + if (heap->cancelled_gc()) { + // It's possible the gc cycle was cancelled after the last time + // the collection checked for cancellation. In which case, the + // old gc cycle is still completed and we have to deal with this + // cancellation. We set the degeneration point to be outside + // the cycle because if this is an allocation failure, that is + // what must be done (there is no degenerated old cycle). If the + // cancellation was due to a heuristic wanting to start a young + // cycle, then we are not actually going to a degenerated cycle, + // so the degenerated point doesn't matter here. + check_cancellation_or_degen(ShenandoahGC::_degenerated_outside_cycle); + } +} + +void ShenandoahControlThread::service_concurrent_cycle(ShenandoahGeneration* generation, GCCause::Cause cause) { // Normal cycle goes via all concurrent phases. If allocation failure (af) happens during // any of the concurrent phases, it first degrades to Degenerated GC and completes GC there. // If second allocation failure happens during Degenerated GC cycle (for example, when GC @@ -393,32 +575,55 @@ void ShenandoahControlThread::service_concurrent_normal_cycle(GCCause::Cause cau if (check_cancellation_or_degen(ShenandoahGC::_degenerated_outside_cycle)) return; GCIdMark gc_id_mark; - ShenandoahGCSession session(cause); + ShenandoahGCSession session(cause, generation); TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); ShenandoahConcurrentGC gc(generation); if (gc.collect(cause)) { // Cycle is complete - heap->heuristics()->record_success_concurrent(); + generation->heuristics()->record_success_concurrent(); heap->shenandoah_policy()->record_success_concurrent(); } else { assert(heap->cancelled_gc(), "Must have been cancelled"); check_cancellation_or_degen(gc.degen_point()); + _degen_generation = generation; } } bool ShenandoahControlThread::check_cancellation_or_degen(ShenandoahGC::ShenandoahDegenPoint point) { ShenandoahHeap* heap = ShenandoahHeap::heap(); - if (heap->cancelled_gc()) { - assert (is_alloc_failure_gc() || in_graceful_shutdown(), "Cancel GC either for alloc failure GC, or gracefully exiting"); - if (!in_graceful_shutdown()) { - assert (_degen_point == ShenandoahGC::_degenerated_outside_cycle, - "Should not be set yet: %s", ShenandoahGC::degen_point_to_string(_degen_point)); - _degen_point = point; - } + if (!heap->cancelled_gc()) { + return false; + } + + if (in_graceful_shutdown()) { return true; } + + assert(_degen_point == ShenandoahGC::_degenerated_outside_cycle, + "Should not be set yet: %s", ShenandoahGC::degen_point_to_string(_degen_point)); + + if (is_alloc_failure_gc()) { + _degen_point = point; + return true; + } + + if (_preemption_requested.is_set()) { + assert(_requested_generation == YOUNG, "Only young GCs may preempt old."); + _preemption_requested.unset(); + + // Old generation marking is only cancellable during concurrent marking. + // Once final mark is complete, the code does not check again for cancellation. + // If old generation was cancelled for an allocation failure, we wouldn't + // make it to this case. The calling code is responsible for forcing a + // cancellation due to allocation failure into a degenerated cycle. + _degen_point = point; + heap->clear_cancelled_gc(false /* clear oom handler */); + return true; + } + + fatal("Cancel GC either for alloc failure GC, or gracefully exiting, or to pause old generation marking."); return false; } @@ -427,28 +632,33 @@ void ShenandoahControlThread::stop_service() { } void ShenandoahControlThread::service_stw_full_cycle(GCCause::Cause cause) { + ShenandoahHeap* const heap = ShenandoahHeap::heap(); + GCIdMark gc_id_mark; - ShenandoahGCSession session(cause); + ShenandoahGCSession session(cause, heap->global_generation()); ShenandoahFullGC gc; gc.collect(cause); - ShenandoahHeap* const heap = ShenandoahHeap::heap(); - heap->heuristics()->record_success_full(); + heap->global_generation()->heuristics()->record_success_full(); heap->shenandoah_policy()->record_success_full(); } void ShenandoahControlThread::service_stw_degenerated_cycle(GCCause::Cause cause, ShenandoahGC::ShenandoahDegenPoint point) { assert (point != ShenandoahGC::_degenerated_unset, "Degenerated point should be set"); + ShenandoahHeap* const heap = ShenandoahHeap::heap(); GCIdMark gc_id_mark; - ShenandoahGCSession session(cause); + ShenandoahGCSession session(cause, _degen_generation); - ShenandoahDegenGC gc(point); + ShenandoahDegenGC gc(point, _degen_generation); gc.collect(cause); - ShenandoahHeap* const heap = ShenandoahHeap::heap(); - heap->heuristics()->record_success_degenerated(); + assert(heap->young_generation()->task_queues()->is_empty(), "Unexpected young generation marking tasks"); + assert(heap->old_generation()->task_queues()->is_empty(), "Unexpected old generation marking tasks"); + assert(heap->global_generation()->task_queues()->is_empty(), "Unexpected global generation marking tasks"); + + _degen_generation->heuristics()->record_success_degenerated(); heap->shenandoah_policy()->record_success_degenerated(); } @@ -480,6 +690,10 @@ bool ShenandoahControlThread::is_explicit_gc(GCCause::Cause cause) const { GCCause::is_serviceability_requested_gc(cause); } +bool ShenandoahControlThread::is_implicit_gc(GCCause::Cause cause) const { + return !is_explicit_gc(cause) && cause != GCCause::_shenandoah_concurrent_gc; +} + void ShenandoahControlThread::request_gc(GCCause::Cause cause) { assert(GCCause::is_user_requested_gc(cause) || GCCause::is_serviceability_requested_gc(cause) || @@ -498,6 +712,34 @@ void ShenandoahControlThread::request_gc(GCCause::Cause cause) { } } +bool ShenandoahControlThread::request_concurrent_gc(GenerationMode generation) { + if (_preemption_requested.is_set() || _gc_requested.is_set() || ShenandoahHeap::heap()->cancelled_gc()) { + // ignore subsequent requests from the heuristics + return false; + } + + if (_mode == none) { + _requested_gc_cause = GCCause::_shenandoah_concurrent_gc; + _requested_generation = generation; + return true; + } + + if (preempt_old_marking(generation)) { + log_info(gc)("Preempting old generation mark to allow young GC."); + _requested_gc_cause = GCCause::_shenandoah_concurrent_gc; + _requested_generation = generation; + _preemption_requested.set(); + ShenandoahHeap::heap()->cancel_gc(GCCause::_shenandoah_concurrent_gc); + return true; + } + + return false; +} + +bool ShenandoahControlThread::preempt_old_marking(GenerationMode generation) { + return generation == YOUNG && _allow_old_preemption.is_set(); +} + void ShenandoahControlThread::handle_requested_gc(GCCause::Cause cause) { // Make sure we have at least one complete GC cycle before unblocking // from the explicit GC request. @@ -595,10 +837,6 @@ void ShenandoahControlThread::notify_heap_changed() { if (_do_counters_update.is_unset()) { _do_counters_update.set(); } - // Notify that something had changed. - if (_heap_changed.is_unset()) { - _heap_changed.set(); - } } void ShenandoahControlThread::pacing_notify_alloc(size_t words) { @@ -643,3 +881,21 @@ void ShenandoahControlThread::prepare_for_graceful_shutdown() { bool ShenandoahControlThread::in_graceful_shutdown() { return _graceful_shutdown.is_set(); } + +const char* ShenandoahControlThread::gc_mode_name(ShenandoahControlThread::GCMode mode) { + switch (mode) { + case none: return "idle"; + case concurrent_normal: return "normal"; + case stw_degenerated: return "degenerated"; + case stw_full: return "full"; + case marking_old: return "old mark"; + default: return "unknown"; + } +} + +void ShenandoahControlThread::set_gc_mode(ShenandoahControlThread::GCMode new_mode) { + if (_mode != new_mode) { + log_info(gc)("Transition from: %s to: %s", gc_mode_name(_mode), gc_mode_name(new_mode)); + _mode = new_mode; + } +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp index 025bc9857bea0..051710b054cf6 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp @@ -57,13 +57,6 @@ class ShenandoahControlThread: public ConcurrentGCThread { friend class VMStructs; private: - typedef enum { - none, - concurrent_normal, - stw_degenerated, - stw_full - } GCMode; - // While we could have a single lock for these, it may risk unblocking // GC waiters when alloc failure GC cycle finishes. We want instead // to make complete explicit cycle for for demanding customers. @@ -73,27 +66,43 @@ class ShenandoahControlThread: public ConcurrentGCThread { ShenandoahPeriodicPacerNotify _periodic_pacer_notify_task; public: + typedef enum { + none, + concurrent_normal, + stw_degenerated, + stw_full, + marking_old + } GCMode; + void run_service(); void stop_service(); + size_t get_gc_id(); + private: + ShenandoahSharedFlag _allow_old_preemption; + ShenandoahSharedFlag _preemption_requested; ShenandoahSharedFlag _gc_requested; ShenandoahSharedFlag _alloc_failure_gc; ShenandoahSharedFlag _graceful_shutdown; - ShenandoahSharedFlag _heap_changed; ShenandoahSharedFlag _do_counters_update; ShenandoahSharedFlag _force_counters_update; GCCause::Cause _requested_gc_cause; + GenerationMode _requested_generation; ShenandoahGC::ShenandoahDegenPoint _degen_point; + ShenandoahGeneration* _degen_generation; shenandoah_padding(0); volatile size_t _allocs_seen; shenandoah_padding(1); volatile size_t _gc_id; shenandoah_padding(2); + volatile GCMode _mode; + shenandoah_padding(3); bool check_cancellation_or_degen(ShenandoahGC::ShenandoahDegenPoint point); - void service_concurrent_normal_cycle(GCCause::Cause cause, ShenandoahGeneration* generation); + void resume_concurrent_old_cycle(ShenandoahGeneration* generation, GCCause::Cause cause); + void service_concurrent_cycle(ShenandoahGeneration* generation, GCCause::Cause cause); void service_stw_full_cycle(GCCause::Cause cause); void service_stw_degenerated_cycle(GCCause::Cause cause, ShenandoahGC::ShenandoahDegenPoint point); void service_uncommit(double shrink_before, size_t shrink_until); @@ -104,7 +113,6 @@ class ShenandoahControlThread: public ConcurrentGCThread { void reset_gc_id(); void update_gc_id(); - size_t get_gc_id(); void notify_gc_waiters(); @@ -113,6 +121,9 @@ class ShenandoahControlThread: public ConcurrentGCThread { void handle_requested_gc(GCCause::Cause cause); bool is_explicit_gc(GCCause::Cause cause) const; + bool is_implicit_gc(GCCause::Cause cause) const; + + bool preempt_old_marking(GenerationMode generation); bool check_soft_max_changed() const; @@ -130,6 +141,7 @@ class ShenandoahControlThread: public ConcurrentGCThread { void handle_alloc_failure_evac(size_t words); void request_gc(GCCause::Cause cause); + bool request_concurrent_gc(GenerationMode generation); void handle_counters_update(); void handle_force_counters_update(); @@ -148,6 +160,21 @@ class ShenandoahControlThread: public ConcurrentGCThread { // Printing void print_on(outputStream* st) const; void print() const; + + void service_concurrent_normal_cycle(const ShenandoahHeap* heap, + const GenerationMode generation, + GCCause::Cause cause); + + void service_concurrent_old_cycle(const ShenandoahHeap* heap, + GCCause::Cause &cause); + + void set_gc_mode(GCMode new_mode); + GCMode gc_mode() { + return _mode; + } + + private: + static const char* gc_mode_name(GCMode mode); }; #endif // SHARE_GC_SHENANDOAH_SHENANDOAHCONTROLTHREAD_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp index 4b0bbecabe87d..94a8aa083f1d8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp @@ -29,6 +29,7 @@ #include "gc/shenandoah/shenandoahConcurrentMark.hpp" #include "gc/shenandoah/shenandoahDegeneratedGC.hpp" #include "gc/shenandoah/shenandoahFullGC.hpp" +#include "gc/shenandoah/shenandoahGeneration.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahMetrics.hpp" #include "gc/shenandoah/shenandoahMonitoringSupport.hpp" @@ -42,9 +43,10 @@ #include "runtime/vmThread.hpp" #include "utilities/events.hpp" -ShenandoahDegenGC::ShenandoahDegenGC(ShenandoahDegenPoint degen_point) : +ShenandoahDegenGC::ShenandoahDegenGC(ShenandoahDegenPoint degen_point, ShenandoahGeneration* generation) : ShenandoahGC(), - _degen_point(degen_point) { + _degen_point(degen_point), + _generation(generation) { } bool ShenandoahDegenGC::collect(GCCause::Cause cause) { @@ -79,7 +81,15 @@ void ShenandoahDegenGC::op_degenerated() { // Degenerated GC is STW, but it can also fail. Current mechanics communicates // GC failure via cancelled_concgc() flag. So, if we detect the failure after // some phase, we have to upgrade the Degenerate GC to Full GC. - heap->clear_cancelled_gc(); + heap->clear_cancelled_gc(true /* clear oom handler */); + + // We can't easily clear the old mark in progress flag because it must be done + // on a safepoint (not sure if that is a hard requirement). At any rate, once + // we are in a degenerated cycle, there should be no more old marking. + if (heap->is_concurrent_old_mark_in_progress()) { + heap->old_generation()->cancel_marking(); + } + assert(heap->old_generation()->task_queues()->is_empty(), "Old gen task queues should be empty."); ShenandoahMetricsSnapshot metrics; metrics.snap_before(); @@ -99,13 +109,12 @@ void ShenandoahDegenGC::op_degenerated() { // Degenerated from concurrent root mark, reset the flag for STW mark if (heap->is_concurrent_mark_in_progress()) { - ShenandoahConcurrentMark::cancel(); - heap->set_concurrent_mark_in_progress(false); + heap->cancel_concurrent_mark(); } // Note that we can only do this for "outside-cycle" degens, otherwise we would risk // changing the cycle parameters mid-cycle during concurrent -> degenerated handover. - heap->set_unload_classes(heap->heuristics()->can_unload_classes()); + heap->set_unload_classes(_generation->heuristics()->can_unload_classes()); op_reset(); @@ -213,20 +222,19 @@ void ShenandoahDegenGC::op_degenerated() { } void ShenandoahDegenGC::op_reset() { - ShenandoahHeap::heap()->prepare_gc(); + _generation->prepare_gc(); } void ShenandoahDegenGC::op_mark() { assert(!ShenandoahHeap::heap()->is_concurrent_mark_in_progress(), "Should be reset"); ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc_stw_mark); - ShenandoahSTWMark mark(false /*full gc*/); - mark.clear(); + ShenandoahSTWMark mark(_generation, false /*full gc*/); mark.mark(); } void ShenandoahDegenGC::op_finish_mark() { - ShenandoahConcurrentMark mark; - mark.finish_mark(ShenandoahHeap::heap()->global_generation()); + ShenandoahConcurrentMark mark(_generation); + mark.finish_mark(); } void ShenandoahDegenGC::op_prepare_evacuation() { @@ -237,8 +245,9 @@ void ShenandoahDegenGC::op_prepare_evacuation() { // STW cleanup weak roots and unload classes heap->parallel_cleaning(false /*full gc*/); + // Prepare regions and collection set - heap->prepare_regions_and_collection_set(false /*concurrent*/); + _generation->prepare_regions_and_collection_set(false /*concurrent*/); // Retire the TLABs, which will force threads to reacquire their TLABs after the pause. // This is needed for two reasons. Strong one: new allocations would be with new freeset, diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.hpp index 8f6f71d52c253..80371b620650a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.hpp @@ -28,14 +28,16 @@ #include "gc/shenandoah/shenandoahGC.hpp" class VM_ShenandoahDegeneratedGC; +class ShenandoahGeneration; class ShenandoahDegenGC : public ShenandoahGC { friend class VM_ShenandoahDegeneratedGC; private: const ShenandoahDegenPoint _degen_point; + ShenandoahGeneration* _generation; public: - ShenandoahDegenGC(ShenandoahDegenPoint degen_point); + ShenandoahDegenGC(ShenandoahDegenPoint degen_point, ShenandoahGeneration* generation); bool collect(GCCause::Cause cause); private: diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index 9aa70f05a2d8c..8d5df95ec6c63 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -24,14 +24,18 @@ #include "precompiled.hpp" #include "gc/shared/tlab_globals.hpp" +#include "gc/shenandoah/shenandoahBarrierSet.hpp" #include "gc/shenandoah/shenandoahFreeSet.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahHeapRegionSet.hpp" #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp" +#include "gc/shenandoah/shenandoahScanRemembered.inline.hpp" +#include "gc/shenandoah/shenandoahYoungGeneration.hpp" #include "logging/logStream.hpp" #include "memory/resourceArea.hpp" #include "runtime/orderAccess.hpp" + ShenandoahFreeSet::ShenandoahFreeSet(ShenandoahHeap* heap, size_t max_regions) : _heap(heap), _mutator_free_bitmap(max_regions, mtGC), @@ -99,8 +103,24 @@ HeapWord* ShenandoahFreeSet::allocate_single(ShenandoahAllocRequest& req, bool& for (size_t c = _collector_rightmost + 1; c > _collector_leftmost; c--) { size_t idx = c - 1; if (is_collector_free(idx)) { - HeapWord* result = try_allocate_in(_heap->get_region(idx), req, in_new_region); + ShenandoahHeapRegion* r = _heap->get_region(idx); + if (r->is_young() && req.is_old()) { + // We don't want to cannibalize a young region to satisfy + // an evacuation from an old region. + continue; + } + HeapWord* result = try_allocate_in(r, req, in_new_region); if (result != NULL) { + if (r->is_old()) { + // HEY! This is a very coarse card marking. We hope to repair + // such cards during remembered set scanning. + + // HEY! To support full generality with alternative remembered set implementations, + // is preferable to not make direct access to the current card_table implementation. + // Try ShenandoahHeap::heap()->card_scan()->mark_range_as_dirty(result, req.actual_size()); + + ShenandoahBarrierSet::barrier_set()->card_table()->dirty_MemRegion(MemRegion(result, req.actual_size())); + } return result; } } @@ -117,9 +137,23 @@ HeapWord* ShenandoahFreeSet::allocate_single(ShenandoahAllocRequest& req, bool& if (is_mutator_free(idx)) { ShenandoahHeapRegion* r = _heap->get_region(idx); if (can_allocate_from(r)) { + if (r->is_young() && req.is_old()) { + continue; + } + flip_to_gc(r); HeapWord *result = try_allocate_in(r, req, in_new_region); if (result != NULL) { + if (r->is_old()) { + // HEY! This is a very coarse card marking. We hope to repair + // such cards during remembered set scanning. + + // HEY! To support full generality with alternative remembered set implementations, + // is preferable to not make direct access to the current card_table implementation. + // Try ShenandoahHeap::heap()->card_scan()->mark_range_as_dirty(result, req.actual_size()); + + ShenandoahBarrierSet::barrier_set()->card_table()->dirty_MemRegion(MemRegion(result, req.actual_size())); + } return result; } } @@ -149,7 +183,9 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah try_recycle_trashed(r); - if (r->affiliation() == FREE) { + if (r->affiliation() == ShenandoahRegionAffiliation::FREE) { + // This free region might have garbage in its remembered set representation. + _heap->clear_cards_for(r); r->set_affiliation(req.affiliation()); } else if (r->affiliation() != req.affiliation()) { return NULL; @@ -185,6 +221,12 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah if (req.is_gc_alloc()) { r->set_update_watermark(r->top()); } + + if (r->affiliation() == ShenandoahRegionAffiliation::YOUNG_GENERATION) { + _heap->young_generation()->increase_used(size * HeapWordSize); + } else if (r->affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) { + _heap->old_generation()->increase_used(size * HeapWordSize); + } } if (result == NULL || has_no_alloc_capacity(r)) { @@ -321,6 +363,12 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req) { // marked used in the free set. increase_used(ShenandoahHeapRegion::region_size_bytes() * num); + if (req.affiliation() == ShenandoahRegionAffiliation::YOUNG_GENERATION) { + _heap->young_generation()->increase_used(ShenandoahHeapRegion::region_size_bytes() * num); + } else if (req.affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) { + _heap->old_generation()->increase_used(ShenandoahHeapRegion::region_size_bytes() * num); + } + if (remainder != 0) { // Record this remainder as allocation waste _heap->notify_mutator_alloc_words(ShenandoahHeapRegion::region_size_words() - remainder, true); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index 5ebbddc561259..0f21734fa6aaa 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -49,6 +49,7 @@ #include "gc/shenandoah/shenandoahVerifier.hpp" #include "gc/shenandoah/shenandoahVMOperations.hpp" #include "gc/shenandoah/shenandoahWorkerPolicy.hpp" +#include "gc/shenandoah/shenandoahYoungGeneration.hpp" #include "memory/metaspace.hpp" #include "memory/universe.hpp" #include "oops/compressedOops.inline.hpp" @@ -95,6 +96,10 @@ void ShenandoahFullGC::entry_full(GCCause::Cause cause) { } void ShenandoahFullGC::op_full(GCCause::Cause cause) { + if (ShenandoahHeap::heap()->mode()->is_generational()) { + fatal("Full GC not yet supported for generational mode."); + } + ShenandoahMetricsSnapshot metrics; metrics.snap_before(); @@ -159,10 +164,9 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) { } assert(!heap->is_update_refs_in_progress(), "sanity"); - // b. Cancel concurrent mark, if in progress + // b. Cancel all concurrent marks, if in progress if (heap->is_concurrent_mark_in_progress()) { - ShenandoahConcurrentGC::cancel(); - heap->set_concurrent_mark_in_progress(false); + heap->cancel_concurrent_mark(); } assert(!heap->is_concurrent_mark_in_progress(), "sanity"); @@ -172,9 +176,9 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) { } // d. Reset the bitmaps for new marking - heap->reset_mark_bitmap(); + heap->global_generation()->reset_mark_bitmap(); assert(heap->marking_context()->is_bitmap_clear(), "sanity"); - assert(!heap->marking_context()->is_complete(), "sanity"); + assert(!heap->global_generation()->is_mark_complete(), "sanity"); // e. Abandon reference discovery and clear all discovered references. ShenandoahReferenceProcessor* rp = heap->ref_processor(); @@ -274,6 +278,8 @@ class ShenandoahPrepareForMarkClosure: public ShenandoahHeapRegionClosure { _ctx->capture_top_at_mark_start(r); r->clear_live_data(); } + + bool is_thread_safe() { return true; } }; void ShenandoahFullGC::phase1_mark_heap() { @@ -283,15 +289,15 @@ void ShenandoahFullGC::phase1_mark_heap() { ShenandoahHeap* heap = ShenandoahHeap::heap(); ShenandoahPrepareForMarkClosure cl; - heap->heap_region_iterate(&cl); + heap->parallel_heap_region_iterate(&cl); - heap->set_unload_classes(heap->heuristics()->can_unload_classes()); + heap->set_unload_classes(heap->global_generation()->heuristics()->can_unload_classes()); ShenandoahReferenceProcessor* rp = heap->ref_processor(); // enable ("weak") refs discovery rp->set_soft_reference_policy(true); // forcefully purge all soft references - ShenandoahSTWMark mark(true /*full_gc*/); + ShenandoahSTWMark mark(heap->global_generation(), true /*full_gc*/); mark.mark(); heap->parallel_cleaning(true /* full_gc */); } @@ -324,6 +330,14 @@ class ShenandoahPrepareForCompactionObjectClosure : public ObjectClosure { void finish_region() { assert(_to_region != NULL, "should not happen"); + if (_heap->mode()->is_generational() && _to_region->affiliation() == FREE) { + // HEY! Changing this region to young during compaction may not be + // technically correct here because it completely disregards the ages + // and origins of the objects being moved. It is, however, certainly + // more correct than putting live objects into a region without a + // generational affiliation. + _to_region->set_affiliation(YOUNG_GENERATION); + } _to_region->set_new_top(_compact_point); } @@ -1061,7 +1075,11 @@ void ShenandoahFullGC::phase4_compact_objects(ShenandoahHeapRegionSet** worker_s heap->collection_set()->clear(); heap->free_set()->rebuild(); + + if (heap->mode()->is_generational()) { + heap->young_generation()->promote_all_regions(); + } } - heap->clear_cancelled_gc(); + heap->clear_cancelled_gc(true /* clear oom handler */); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp new file mode 100644 index 0000000000000..f6f46f047a58e --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -0,0 +1,276 @@ +/* + * Copyright (c) 2020, 2021 Amazon.com, Inc. and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "gc/shenandoah/shenandoahFreeSet.hpp" +#include "gc/shenandoah/shenandoahGeneration.hpp" +#include "gc/shenandoah/shenandoahHeap.hpp" +#include "gc/shenandoah/shenandoahMarkClosures.hpp" +#include "gc/shenandoah/shenandoahMonitoringSupport.hpp" +#include "gc/shenandoah/shenandoahTaskqueue.inline.hpp" +#include "gc/shenandoah/shenandoahUtils.hpp" +#include "gc/shenandoah/shenandoahVerifier.hpp" +#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" + +class ShenandoahResetUpdateRegionStateClosure : public ShenandoahHeapRegionClosure { + private: + ShenandoahMarkingContext* const _ctx; + public: + ShenandoahResetUpdateRegionStateClosure() : + _ctx(ShenandoahHeap::heap()->marking_context()) {} + + void heap_region_do(ShenandoahHeapRegion* r) { + if (r->is_active()) { + // Reset live data and set TAMS optimistically. We would recheck these under the pause + // anyway to capture any updates that happened since now. + _ctx->capture_top_at_mark_start(r); + r->clear_live_data(); + } + } + + bool is_thread_safe() { return true; } +}; + +class ShenandoahResetBitmapTask : public ShenandoahHeapRegionClosure { + private: + ShenandoahHeap* _heap; + ShenandoahMarkingContext* const _ctx; + public: + ShenandoahResetBitmapTask() : + _heap(ShenandoahHeap::heap()), + _ctx(_heap->marking_context()) {} + + void heap_region_do(ShenandoahHeapRegion* region) { + if (_heap->is_bitmap_slice_committed(region)) { + _ctx->clear_bitmap(region); + } + } + + bool is_thread_safe() { return true; } +}; + +void ShenandoahGeneration::initialize_heuristics(ShenandoahMode* gc_mode) { + _heuristics = gc_mode->initialize_heuristics(this); + + if (_heuristics->is_diagnostic() && !UnlockDiagnosticVMOptions) { + vm_exit_during_initialization( + err_msg("Heuristics \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.", + _heuristics->name())); + } + if (_heuristics->is_experimental() && !UnlockExperimentalVMOptions) { + vm_exit_during_initialization( + err_msg("Heuristics \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.", + _heuristics->name())); + } +} + +size_t ShenandoahGeneration::bytes_allocated_since_gc_start() { + return ShenandoahHeap::heap()->bytes_allocated_since_gc_start(); +} + +void ShenandoahGeneration::log_status() const { + typedef LogTarget(Info, gc, ergo) LogGcInfo; + + if (!LogGcInfo::is_enabled()) { + return; + } + + // Not under a lock here, so read each of these once to make sure + // byte size in proper unit and proper unit for byte size are consistent. + size_t v_used = used(); + size_t v_used_regions = used_regions_size(); + size_t v_soft_max_capacity = soft_max_capacity(); + size_t v_max_capacity = max_capacity(); + size_t v_available = available(); + LogGcInfo::print("%s generation used: " SIZE_FORMAT "%s, used regions: " SIZE_FORMAT "%s, " + "soft capacity: " SIZE_FORMAT "%s, max capacity: " SIZE_FORMAT " %s, available: " SIZE_FORMAT " %s", + name(), + byte_size_in_proper_unit(v_used), proper_unit_for_byte_size(v_used), + byte_size_in_proper_unit(v_used_regions), proper_unit_for_byte_size(v_used_regions), + byte_size_in_proper_unit(v_soft_max_capacity), proper_unit_for_byte_size(v_soft_max_capacity), + byte_size_in_proper_unit(v_max_capacity), proper_unit_for_byte_size(v_max_capacity), + byte_size_in_proper_unit(v_available), proper_unit_for_byte_size(v_available)); +} + +void ShenandoahGeneration::reset_mark_bitmap() { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + heap->assert_gc_workers(heap->workers()->active_workers()); + + set_mark_incomplete(); + + ShenandoahResetBitmapTask task; + parallel_heap_region_iterate(&task); +} + +void ShenandoahGeneration::prepare_gc() { + reset_mark_bitmap(); + + ShenandoahResetUpdateRegionStateClosure cl; + parallel_heap_region_iterate(&cl); +} + +void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + assert(!heap->is_full_gc_in_progress(), "Only for concurrent and degenerated GC"); + { + ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_update_region_states : + ShenandoahPhaseTimings::degen_gc_final_update_region_states); + ShenandoahFinalMarkUpdateRegionStateClosure cl(complete_marking_context()); + parallel_heap_region_iterate(&cl); + + heap->assert_pinned_region_status(); + } + + { + ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::choose_cset : + ShenandoahPhaseTimings::degen_gc_choose_cset); + ShenandoahHeapLocker locker(heap->lock()); + heap->collection_set()->clear(); + _heuristics->choose_collection_set(heap->collection_set()); + } + + { + ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_rebuild_freeset : + ShenandoahPhaseTimings::degen_gc_final_rebuild_freeset); + ShenandoahHeapLocker locker(heap->lock()); + heap->free_set()->rebuild(); + } +} + +bool ShenandoahGeneration::is_bitmap_clear() { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + ShenandoahMarkingContext* context = heap->marking_context(); + size_t num_regions = heap->num_regions(); + for (size_t idx = 0; idx < num_regions; idx++) { + ShenandoahHeapRegion* r = heap->get_region(idx); + if (contains(r)) { + if (heap->is_bitmap_slice_committed(r) && !context->is_bitmap_clear_range(r->bottom(), r->end())) { + return false; + } + } + } + return true; +} + +bool ShenandoahGeneration::is_mark_complete() { + return _is_marking_complete.is_set(); +} + +void ShenandoahGeneration::set_mark_complete() { + _is_marking_complete.set(); +} + +void ShenandoahGeneration::set_mark_incomplete() { + _is_marking_complete.unset(); +} + +ShenandoahMarkingContext* ShenandoahGeneration::complete_marking_context() { + assert(is_mark_complete(), "Marking must be completed."); + return ShenandoahHeap::heap()->marking_context(); +} + +void ShenandoahGeneration::cancel_marking() { + if (is_concurrent_mark_in_progress()) { + set_concurrent_mark_in_progress(false); + } + set_mark_incomplete(); + _task_queues->clear(); +} + +ShenandoahGeneration::ShenandoahGeneration(GenerationMode generation_mode, + uint max_queues, + size_t max_capacity, + size_t soft_max_capacity) : + _generation_mode(generation_mode), + _task_queues(new ShenandoahObjToScanQueueSet(max_queues)), + _affiliated_region_count(0), _used(0), + _max_capacity(max_capacity), _soft_max_capacity(soft_max_capacity) { + _is_marking_complete.set(); + assert(max_queues > 0, "At least one queue"); + for (uint i = 0; i < max_queues; ++i) { + ShenandoahObjToScanQueue* task_queue = new ShenandoahObjToScanQueue(); + task_queue->initialize(); + _task_queues->register_queue(i, task_queue); + } +} + +ShenandoahGeneration::~ShenandoahGeneration() { + for (uint i = 0; i < _task_queues->size(); ++i) { + ShenandoahObjToScanQueue* q = _task_queues->queue(i); + delete q; + } + delete _task_queues; +} + +void ShenandoahGeneration::reserve_task_queues(uint workers) { + _task_queues->reserve(workers); +} + +ShenandoahObjToScanQueueSet* ShenandoahGeneration::old_gen_task_queues() const { + return nullptr; +} + +void ShenandoahGeneration::scan_remembered_set() { + shenandoah_assert_safepoint(); + assert(generation_mode() == YOUNG, "Should only scan remembered set for young generation."); + + ShenandoahHeap* const heap = ShenandoahHeap::heap(); + // TODO: Add a phase for rset scan. + // ShenandoahGCPhase phase(ShenandoahPhaseTimings::finish_mark); + uint nworkers = heap->workers()->active_workers(); + reserve_task_queues(nworkers); + + ShenandoahReferenceProcessor* rp = heap->ref_processor(); + ShenandoahRegionIterator regions; + ShenandoahScanRememberedTask task(task_queues(), old_gen_task_queues(), rp, ®ions); + heap->workers()->run_task(&task); +} + +void ShenandoahGeneration::increment_affiliated_region_count() { + _affiliated_region_count++; +} + +void ShenandoahGeneration::decrement_affiliated_region_count() { + _affiliated_region_count--; +} + +void ShenandoahGeneration::increase_used(size_t bytes) { + Atomic::add(&_used, bytes); +} + +void ShenandoahGeneration::decrease_used(size_t bytes) { + assert(_used >= bytes, "cannot reduce bytes used by generation below zero"); + Atomic::sub(&_used, bytes); +} + +size_t ShenandoahGeneration::used_regions_size() const { + return _affiliated_region_count * ShenandoahHeapRegion::region_size_bytes(); +} + +size_t ShenandoahGeneration::available() const { + size_t in_use = used(); + size_t soft_capacity = soft_max_capacity(); + return in_use > soft_capacity ? 0 : soft_capacity - in_use; +} \ No newline at end of file diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index e1023e5c49be9..bbfdb5becc3b5 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, Amazon.com, Inc. and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2021 Amazon.com, Inc. and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -26,18 +26,101 @@ #define SHARE_VM_GC_SHENANDOAH_SHENANDOAHGENERATION_HPP #include "memory/allocation.hpp" -#include "gc/shenandoah/shenandoahConcurrentMark.hpp" +#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" +#include "gc/shenandoah/mode/shenandoahGenerationalMode.hpp" +#include "gc/shenandoah/shenandoahLock.hpp" +#include "gc/shenandoah/shenandoahMarkingContext.hpp" + +class ShenandoahHeapRegion; +class ShenandoahHeapRegionClosure; class ShenandoahGeneration : public CHeapObj { private: GenerationMode const _generation_mode; + ShenandoahHeuristics* _heuristics; + + // Marking task queues and completeness + ShenandoahObjToScanQueueSet* _task_queues; + ShenandoahSharedFlag _is_marking_complete; + +protected: + // Usage + size_t _affiliated_region_count; + volatile size_t _used; + size_t _max_capacity; + size_t _soft_max_capacity; public: - explicit ShenandoahGeneration(GenerationMode generation_mode) : - _generation_mode(generation_mode) { - } + ShenandoahGeneration(GenerationMode generation_mode, uint max_queues, size_t max_capacity, size_t soft_max_capacity); + ~ShenandoahGeneration(); inline GenerationMode generation_mode() const { return _generation_mode; } + + inline ShenandoahHeuristics* heuristics() const { return _heuristics; } + + virtual const char* name() const = 0; + + void initialize_heuristics(ShenandoahMode* gc_mode); + + virtual size_t soft_max_capacity() const { return _soft_max_capacity; } + virtual size_t max_capacity() const { return _max_capacity; } + virtual size_t used_regions_size() const; + virtual size_t used() const { return _used; } + virtual size_t available() const; + + void set_soft_max_capacity(size_t soft_max_capacity) { + _soft_max_capacity = soft_max_capacity; + } + + virtual size_t bytes_allocated_since_gc_start(); + + void log_status() const; + + // Used directly by FullGC + void reset_mark_bitmap(); + + // Used by concurrent and degenerated GC to reset regions. + void prepare_gc(); + void prepare_regions_and_collection_set(bool concurrent); + + // Cancel marking (used by Full collect and when cancelling cycle). + void cancel_marking(); + + // Return true if this region is affiliated with this generation. + virtual bool contains(ShenandoahHeapRegion* region) const = 0; + + // Apply closure to all regions affiliated with this generation. + virtual void parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl) = 0; + + // This is public to support cancellation of marking when a Full cycle is started. + virtual void set_concurrent_mark_in_progress(bool in_progress) = 0; + + // Check the bitmap only for regions belong to this generation. + bool is_bitmap_clear(); + + // We need to track the status of marking for different generations. + bool is_mark_complete(); + void set_mark_complete(); + void set_mark_incomplete(); + + ShenandoahMarkingContext* complete_marking_context(); + + // Task queues + ShenandoahObjToScanQueueSet* task_queues() const { return _task_queues; } + virtual void reserve_task_queues(uint workers); + virtual ShenandoahObjToScanQueueSet* old_gen_task_queues() const; + + void scan_remembered_set(); + + void increment_affiliated_region_count(); + void decrement_affiliated_region_count(); + + void increase_used(size_t bytes); + void decrease_used(size_t bytes); + + protected: + + virtual bool is_concurrent_mark_in_progress() = 0; }; #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHGENERATION_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp new file mode 100644 index 0000000000000..5869d19109a43 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2020, Red Hat, Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ +#include "precompiled.hpp" + +#include "gc/shenandoah/shenandoahFreeSet.hpp" +#include "gc/shenandoah/shenandoahGlobalGeneration.hpp" +#include "gc/shenandoah/shenandoahHeap.hpp" +#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" +#include "gc/shenandoah/shenandoahUtils.hpp" +#include "gc/shenandoah/shenandoahVerifier.hpp" +#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" + +const char* ShenandoahGlobalGeneration::name() const { + return "GLOBAL"; +} + +size_t ShenandoahGlobalGeneration::max_capacity() const { + return ShenandoahHeap::heap()->max_capacity(); +} + +size_t ShenandoahGlobalGeneration::used_regions_size() const { + return ShenandoahHeap::heap()->capacity(); +} + +size_t ShenandoahGlobalGeneration::soft_max_capacity() const { + return ShenandoahHeap::heap()->soft_max_capacity(); +} + +size_t ShenandoahGlobalGeneration::used() const { + return ShenandoahHeap::heap()->used(); +} + +size_t ShenandoahGlobalGeneration::available() const { + return ShenandoahHeap::heap()->free_set()->available(); +} + +void ShenandoahGlobalGeneration::set_concurrent_mark_in_progress(bool in_progress) { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + if (in_progress && heap->is_concurrent_old_mark_in_progress()) { + // Global collection has preempted an old generation mark. This is fine + // because the global generation includes the old generation, but we + // want the global collect to start from a clean slate and we don't want + // any stale state in the old generation. + heap->purge_old_satb_buffers(true /* abandon */); + heap->old_generation()->cancel_marking(); + } + + heap->set_concurrent_young_mark_in_progress(in_progress); + heap->set_concurrent_old_mark_in_progress(in_progress); +} + +bool ShenandoahGlobalGeneration::contains(ShenandoahHeapRegion* region) const { + return true; +} + +void ShenandoahGlobalGeneration::parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl) { + ShenandoahHeap::heap()->parallel_heap_region_iterate(cl); +} + +bool ShenandoahGlobalGeneration::is_concurrent_mark_in_progress() { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + return heap->is_concurrent_mark_in_progress(); +} + diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp index 224c5f25d7bf2..4355807bb1145 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp @@ -30,7 +30,26 @@ // A "generation" that represents the whole heap. class ShenandoahGlobalGeneration : public ShenandoahGeneration { public: - ShenandoahGlobalGeneration() : ShenandoahGeneration(GLOBAL) { } + ShenandoahGlobalGeneration(uint max_queues) + : ShenandoahGeneration(GLOBAL, max_queues, 0, 0) { } + +public: + virtual const char* name() const; + + virtual size_t max_capacity() const; + virtual size_t soft_max_capacity() const; + virtual size_t used_regions_size() const; + virtual size_t used() const; + virtual size_t available() const; + + virtual void set_concurrent_mark_in_progress(bool in_progress); + + bool contains(ShenandoahHeapRegion* region) const; + + void parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl); + + protected: + bool is_concurrent_mark_in_progress(); }; #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHGLOBALGENERATION_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 8c33ef8dccd5c..86b592d97b56c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -41,6 +41,7 @@ #include "gc/shenandoah/shenandoahCollectorPolicy.hpp" #include "gc/shenandoah/shenandoahConcurrentMark.hpp" #include "gc/shenandoah/shenandoahControlThread.hpp" +#include "gc/shenandoah/shenandoahRegulatorThread.hpp" #include "gc/shenandoah/shenandoahFreeSet.hpp" #include "gc/shenandoah/shenandoahGlobalGeneration.hpp" #include "gc/shenandoah/shenandoahPhaseTimings.hpp" @@ -52,6 +53,7 @@ #include "gc/shenandoah/shenandoahMemoryPool.hpp" #include "gc/shenandoah/shenandoahMetrics.hpp" #include "gc/shenandoah/shenandoahMonitoringSupport.hpp" +#include "gc/shenandoah/shenandoahOldGeneration.hpp" #include "gc/shenandoah/shenandoahOopClosures.inline.hpp" #include "gc/shenandoah/shenandoahPacer.inline.hpp" #include "gc/shenandoah/shenandoahPadding.hpp" @@ -159,9 +161,6 @@ jint ShenandoahHeap::initialize() { _num_regions = ShenandoahHeapRegion::region_count(); - // Now we know the number of regions, initialize the heuristics. - initialize_heuristics(); - size_t num_committed_regions = init_byte_size / reg_size_bytes; num_committed_regions = MIN2(num_committed_regions, _num_regions); assert(num_committed_regions <= _num_regions, "sanity"); @@ -177,6 +176,10 @@ jint ShenandoahHeap::initialize() { _committed = _initial_size; + // Now we know the number of regions and heap sizes, initialize the heuristics. + initialize_generations(); + initialize_heuristics(); + size_t heap_page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size(); size_t bitmap_page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size(); size_t region_page_size = UseLargePages ? (size_t)os::large_page_size() : (size_t)os::vm_page_size(); @@ -283,7 +286,7 @@ jint ShenandoahHeap::initialize() { "Cannot commit bitmap memory"); } - _marking_context = new ShenandoahMarkingContext(_heap_region, _bitmap_region, _num_regions, _max_workers); + _marking_context = new ShenandoahMarkingContext(_heap_region, _bitmap_region, _num_regions); if (ShenandoahVerify) { ReservedSpace verify_bitmap(_bitmap_size, bitmap_page_size); @@ -430,12 +433,24 @@ jint ShenandoahHeap::initialize() { } _control_thread = new ShenandoahControlThread(); + _regulator_thread = new ShenandoahRegulatorThread(_control_thread); ShenandoahInitLogger::print(); return JNI_OK; } +void ShenandoahHeap::initialize_generations() { + size_t max_capacity_new = young_generation_capacity(max_capacity()); + size_t soft_max_capacity_new = young_generation_capacity(soft_max_capacity()); + size_t max_capacity_old = max_capacity() - max_capacity_new; + size_t soft_max_capacity_old = soft_max_capacity() - soft_max_capacity_new; + + _young_generation = new ShenandoahYoungGeneration(_max_workers, max_capacity_new, soft_max_capacity_new); + _old_generation = new ShenandoahOldGeneration(_max_workers, max_capacity_old, soft_max_capacity_old); + _global_generation = new ShenandoahGlobalGeneration(_max_workers); +} + void ShenandoahHeap::initialize_heuristics() { if (ShenandoahGCMode != NULL) { if (strcmp(ShenandoahGCMode, "satb") == 0) { @@ -464,18 +479,9 @@ void ShenandoahHeap::initialize_heuristics() { _gc_mode->name())); } - _heuristics = _gc_mode->initialize_heuristics(); - - if (_heuristics->is_diagnostic() && !UnlockDiagnosticVMOptions) { - vm_exit_during_initialization( - err_msg("Heuristics \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.", - _heuristics->name())); - } - if (_heuristics->is_experimental() && !UnlockExperimentalVMOptions) { - vm_exit_during_initialization( - err_msg("Heuristics \"%s\" is experimental, and must be enabled via -XX:+UnlockExperimentalVMOptions.", - _heuristics->name())); - } + _global_generation->initialize_heuristics(_gc_mode); + _young_generation->initialize_heuristics(_gc_mode); + _old_generation->initialize_heuristics(_gc_mode); } #ifdef _MSC_VER @@ -485,6 +491,7 @@ void ShenandoahHeap::initialize_heuristics() { ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) : CollectedHeap(), + _gc_generation(NULL), _initial_size(0), _used(0), _committed(0), @@ -496,11 +503,13 @@ ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) : _num_regions(0), _regions(NULL), _update_refs_iterator(this), - _young_generation(new ShenandoahYoungGeneration()), - _global_generation(new ShenandoahGlobalGeneration()), + _cancel_requested_time(0), + _young_generation(NULL), + _global_generation(NULL), + _old_generation(NULL), _control_thread(NULL), + _regulator_thread(NULL), _shenandoah_policy(policy), - _heuristics(NULL), _free_set(NULL), _pacer(NULL), _verifier(NULL), @@ -529,35 +538,6 @@ ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) : #pragma warning( pop ) #endif -class ShenandoahResetBitmapTask : public AbstractGangTask { -private: - ShenandoahRegionIterator _regions; - -public: - ShenandoahResetBitmapTask() : - AbstractGangTask("Shenandoah Reset Bitmap") {} - - void work(uint worker_id) { - ShenandoahHeapRegion* region = _regions.next(); - ShenandoahHeap* heap = ShenandoahHeap::heap(); - ShenandoahMarkingContext* const ctx = heap->marking_context(); - while (region != NULL) { - if (heap->is_bitmap_slice_committed(region)) { - ctx->clear_bitmap(region); - } - region = _regions.next(); - } - } -}; - -void ShenandoahHeap::reset_mark_bitmap() { - assert_gc_workers(_workers->active_workers()); - mark_incomplete_marking_context(); - - ShenandoahResetBitmapTask task; - _workers->run_task(&task); -} - void ShenandoahHeap::print_on(outputStream* st) const { st->print_cr("Shenandoah Heap"); st->print_cr(" " SIZE_FORMAT "%s max, " SIZE_FORMAT "%s soft max, " SIZE_FORMAT "%s committed, " SIZE_FORMAT "%s used", @@ -572,7 +552,8 @@ void ShenandoahHeap::print_on(outputStream* st) const { st->print("Status: "); if (has_forwarded_objects()) st->print("has forwarded objects, "); - if (is_concurrent_mark_in_progress()) st->print("marking, "); + if (is_concurrent_old_mark_in_progress()) st->print("old marking, "); + if (is_concurrent_young_mark_in_progress()) st->print("young marking, "); if (is_evacuation_in_progress()) st->print("evacuating, "); if (is_update_refs_in_progress()) st->print("updating refs, "); if (is_degenerated_gc_in_progress()) st->print("degenerated gc, "); @@ -635,11 +616,39 @@ void ShenandoahHeap::post_initialize() { _safepoint_workers->set_initialize_gclab(); } - _heuristics->initialize(); - JFR_ONLY(ShenandoahJFRSupport::register_jfr_type_serializers()); } +bool ShenandoahHeap::is_gc_generation_young() const { + return _gc_generation != NULL && _gc_generation->generation_mode() == YOUNG; +} + +// There are three JVM parameters for setting young gen capacity: +// NewSize, MaxNewSize, NewRatio. +// +// If only NewSize is set, it assigns a fixed size and the other two parameters are ignored. +// Otherwise NewRatio applies. +// +// If NewSize is set in any combination, it provides a lower bound. +// +// If MaxNewSize is set it provides an upper bound. +// If this bound is smaller than NewSize, it supersedes, +// resulting in a fixed size given by MaxNewSize. +size_t ShenandoahHeap::young_generation_capacity(size_t capacity) { + if (FLAG_IS_CMDLINE(NewSize) && !FLAG_IS_CMDLINE(MaxNewSize) && !FLAG_IS_CMDLINE(NewRatio)) { + capacity = MIN2(NewSize, capacity); + } else { + capacity /= NewRatio + 1; + if (FLAG_IS_CMDLINE(NewSize)) { + capacity = MAX2(NewSize, capacity); + } + if (FLAG_IS_CMDLINE(MaxNewSize)) { + capacity = MIN2(MaxNewSize, capacity); + } + } + return capacity; +} + size_t ShenandoahHeap::used() const { return Atomic::load_acquire(&_used); } @@ -711,6 +720,13 @@ void ShenandoahHeap::set_soft_max_capacity(size_t v) { "Should be in bounds: " SIZE_FORMAT " <= " SIZE_FORMAT " <= " SIZE_FORMAT, min_capacity(), v, max_capacity()); Atomic::store(&_soft_max_size, v); + + if (mode()->is_generational()) { + size_t soft_max_capacity_young = young_generation_capacity(_soft_max_size); + size_t soft_max_capacity_old = _soft_max_size - soft_max_capacity_young; + _young_generation->set_soft_max_capacity(soft_max_capacity_young); + _old_generation->set_soft_max_capacity(soft_max_capacity_old); + } } size_t ShenandoahHeap::min_capacity() const { @@ -728,7 +744,11 @@ bool ShenandoahHeap::is_in(const void* p) const { } bool ShenandoahHeap::is_in_young(const void* p) const { - return heap_region_containing(p)->affiliation() == YOUNG_GENERATION; + return heap_region_containing(p)->affiliation() == ShenandoahRegionAffiliation::YOUNG_GENERATION; +} + +bool ShenandoahHeap::is_in_old(const void* p) const { + return heap_region_containing(p)->affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION; } void ShenandoahHeap::op_uncommit(double shrink_before, size_t shrink_until) { @@ -758,6 +778,7 @@ void ShenandoahHeap::op_uncommit(double shrink_before, size_t shrink_until) { if (count > 0) { control_thread()->notify_heap_changed(); + regulator_thread()->notify_heap_changed(); } } @@ -884,6 +905,7 @@ HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) { if (in_new_region) { control_thread()->notify_heap_changed(); + regulator_thread()->notify_heap_changed(); } if (result != NULL) { @@ -913,7 +935,43 @@ HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) { HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req, bool& in_new_region) { ShenandoahHeapLocker locker(lock()); - return _free_set->allocate(req, in_new_region); + HeapWord* result = _free_set->allocate(req, in_new_region); + // Register the newly allocated object while we're holding the global lock since there's no synchronization + // built in to the implementation of register_object(). There are potential races when multiple independent + // threads are allocating objects, some of which might span the same card region. For example, consider + // a card table's memory region within which three objects are being allocated by three different threads: + // + // objects being "concurrently" allocated: + // [-----a------][-----b-----][--------------c------------------] + // [---- card table memory range --------------] + // + // Before any objects are allocated, this card's memory range holds no objects. Note that: + // allocation of object a wants to set the has-object, first-start, and last-start attributes of the preceding card region. + // allocation of object b wants to set the has-object, first-start, and last-start attributes of this card region. + // allocation of object c also wants to set the has-object, first-start, and last-start attributes of this card region. + // + // The thread allocating b and the thread allocating c can "race" in various ways, resulting in confusion, such as last-start + // representing object b while first-start represents object c. This is why we need to require all register_object() + // invocations to be "mutually exclusive". Later, when we use GCLABs to allocate memory for promotions and evacuations, + // the protocol may work something like the following: + // 1. The GCLAB is allocated by this (or similar) function, while holding the global lock. + // 2. The GCLAB is registered as a single object. + /// 3. The GCLAB is always aligned at the start of a card memory range and is always a multiple of the card-table memory range size + // 3. Individual allocations carved from the GCLAB are not immediately registered + // 4. When the GCLAB is eventually retired, all of the objects allocated within the GCLAB are registered in batch by a + // single thread. No further synchronization is required because no other allocations will pertain to the same + // card-table memory ranges. + // + // The other case that needs special handling is promotion of regions en masse. When the region is promoted, all objects contained + // within the region are registered. Since the region is a multiple of card-table memory range sizes, there is no need for + // synchronization. It might be nice to figure out how to allow multiple threads to work together to register all of the objects in + // a promoted region, or at least try to balance the efforts so that different gc threads work on registering the objects of + // different heap regions. But that effort will come later. + // + if (result != NULL && req.affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) { + ShenandoahHeap::heap()->card_scan()->register_object(result); + } + return result; } HeapWord* ShenandoahHeap::mem_allocate(size_t size, @@ -928,8 +986,8 @@ MetaWord* ShenandoahHeap::satisfy_failed_metadata_allocation(ClassLoaderData* lo MetaWord* result; // Inform metaspace OOM to GC heuristics if class unloading is possible. - if (heuristics()->can_unload_classes()) { - ShenandoahHeuristics* h = heuristics(); + ShenandoahHeuristics* h = global_generation()->heuristics(); + if (h->can_unload_classes()) { h->record_metaspace_oom(); } @@ -1007,7 +1065,8 @@ class ShenandoahEvacuationTask : public AbstractGangTask { ShenandoahConcurrentEvacuateRegionObjectClosure cl(_sh); ShenandoahHeapRegion* r; while ((r =_cs->claim_next()) != NULL) { - assert(r->has_live(), "Region " SIZE_FORMAT " should have been reclaimed early", r->index()); + // Generational mode doesn't support immediate collection + assert(_sh->mode()->is_generational() || r->has_live(), "Region " SIZE_FORMAT " should have been reclaimed early", r->index()); _sh->marked_object_iterate(r, &cl); if (ShenandoahPacing) { @@ -1547,29 +1606,6 @@ void ShenandoahHeap::parallel_heap_region_iterate(ShenandoahHeapRegionClosure* b } } -class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure { -private: - ShenandoahMarkingContext* const _ctx; -public: - ShenandoahInitMarkUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {} - - void heap_region_do(ShenandoahHeapRegion* r) { - assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index()); - if (r->is_active()) { - // Check if region needs updating its TAMS. We have updated it already during concurrent - // reset, so it is very likely we don't need to do another write here. - if (_ctx->top_at_mark_start(r) != r->top()) { - _ctx->capture_top_at_mark_start(r); - } - } else { - assert(_ctx->top_at_mark_start(r) == r->top(), - "Region " SIZE_FORMAT " should already have correct TAMS", r->index()); - } - } - - bool is_thread_safe() { return true; } -}; - void ShenandoahHeap::rendezvous_threads() { ShenandoahRendezvousClosure cl; Handshake::execute(&cl); @@ -1579,105 +1615,6 @@ void ShenandoahHeap::recycle_trash() { free_set()->recycle_trash(); } -class ShenandoahResetUpdateRegionStateClosure : public ShenandoahHeapRegionClosure { -private: - ShenandoahMarkingContext* const _ctx; -public: - ShenandoahResetUpdateRegionStateClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {} - - void heap_region_do(ShenandoahHeapRegion* r) { - if (r->is_active()) { - // Reset live data and set TAMS optimistically. We would recheck these under the pause - // anyway to capture any updates that happened since now. - r->clear_live_data(); - _ctx->capture_top_at_mark_start(r); - } - } - - bool is_thread_safe() { return true; } -}; - -void ShenandoahHeap::prepare_gc() { - reset_mark_bitmap(); - - ShenandoahResetUpdateRegionStateClosure cl; - parallel_heap_region_iterate(&cl); -} - -class ShenandoahFinalMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure { -private: - ShenandoahMarkingContext* const _ctx; - ShenandoahHeapLock* const _lock; - -public: - ShenandoahFinalMarkUpdateRegionStateClosure() : - _ctx(ShenandoahHeap::heap()->complete_marking_context()), _lock(ShenandoahHeap::heap()->lock()) {} - - void heap_region_do(ShenandoahHeapRegion* r) { - if (r->is_active()) { - // All allocations past TAMS are implicitly live, adjust the region data. - // Bitmaps/TAMS are swapped at this point, so we need to poll complete bitmap. - HeapWord *tams = _ctx->top_at_mark_start(r); - HeapWord *top = r->top(); - if (top > tams) { - r->increase_live_data_alloc_words(pointer_delta(top, tams)); - } - - // We are about to select the collection set, make sure it knows about - // current pinning status. Also, this allows trashing more regions that - // now have their pinning status dropped. - if (r->is_pinned()) { - if (r->pin_count() == 0) { - ShenandoahHeapLocker locker(_lock); - r->make_unpinned(); - } - } else { - if (r->pin_count() > 0) { - ShenandoahHeapLocker locker(_lock); - r->make_pinned(); - } - } - - // Remember limit for updating refs. It's guaranteed that we get no - // from-space-refs written from here on. - r->set_update_watermark_at_safepoint(r->top()); - } else { - assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index()); - assert(_ctx->top_at_mark_start(r) == r->top(), - "Region " SIZE_FORMAT " should have correct TAMS", r->index()); - } - } - - bool is_thread_safe() { return true; } -}; - -void ShenandoahHeap::prepare_regions_and_collection_set(bool concurrent) { - assert(!is_full_gc_in_progress(), "Only for concurrent and degenerated GC"); - { - ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_update_region_states : - ShenandoahPhaseTimings::degen_gc_final_update_region_states); - ShenandoahFinalMarkUpdateRegionStateClosure cl; - parallel_heap_region_iterate(&cl); - - assert_pinned_region_status(); - } - - { - ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::choose_cset : - ShenandoahPhaseTimings::degen_gc_choose_cset); - ShenandoahHeapLocker locker(lock()); - _collection_set->clear(); - heuristics()->choose_collection_set(_collection_set); - } - - { - ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_rebuild_freeset : - ShenandoahPhaseTimings::degen_gc_final_rebuild_freeset); - ShenandoahHeapLocker locker(lock()); - _free_set->rebuild(); - } -} - void ShenandoahHeap::do_class_unloading() { _unloader.unload(); set_concurrent_weak_root_in_progress(false); @@ -1720,13 +1657,40 @@ void ShenandoahHeap::set_gc_state_mask(uint mask, bool value) { set_gc_state_all_threads(_gc_state.raw_value()); } -void ShenandoahHeap::set_concurrent_mark_in_progress(bool in_progress) { +void ShenandoahHeap::set_concurrent_young_mark_in_progress(bool in_progress) { + if (has_forwarded_objects()) { + set_gc_state_mask(YOUNG_MARKING | UPDATEREFS, in_progress); + } else { + set_gc_state_mask(YOUNG_MARKING, in_progress); + } + + manage_satb_barrier(in_progress); +} + +void ShenandoahHeap::set_concurrent_old_mark_in_progress(bool in_progress) { if (has_forwarded_objects()) { - set_gc_state_mask(MARKING | UPDATEREFS, in_progress); + set_gc_state_mask(OLD_MARKING | UPDATEREFS, in_progress); } else { - set_gc_state_mask(MARKING, in_progress); + set_gc_state_mask(OLD_MARKING, in_progress); + } + + manage_satb_barrier(in_progress); +} + +void ShenandoahHeap::manage_satb_barrier(bool active) { + if (is_concurrent_mark_in_progress()) { + // Ignore request to deactivate barrier while concurrent mark is in progress. + // Do not attempt to re-activate the barrier if it is already active. + if (active && !ShenandoahBarrierSet::satb_mark_queue_set().is_active()) { + ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(active, !active); + } + } else { + // No concurrent marking is in progress so honor request to deactivate, + // but only if the barrier is already active. + if (!active && ShenandoahBarrierSet::satb_mark_queue_set().is_active()) { + ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(active, !active); + } } - ShenandoahBarrierSet::satb_mark_queue_set().set_active_all_threads(in_progress, !in_progress); } void ShenandoahHeap::set_evacuation_in_progress(bool in_progress) { @@ -1775,11 +1739,23 @@ bool ShenandoahHeap::try_cancel_gc() { } } +void ShenandoahHeap::cancel_concurrent_mark() { + _young_generation->cancel_marking(); + _old_generation->cancel_marking(); + _global_generation->cancel_marking(); + + ShenandoahBarrierSet::satb_mark_queue_set().abandon_partial_marking(); + + // HEY! Previously, only ShenandoahConcurrentMark::cancel (static) cleared ref processor. + ref_processor()->abandon_partial_discovery(); +} + void ShenandoahHeap::cancel_gc(GCCause::Cause cause) { if (try_cancel_gc()) { FormatBuffer<> msg("Cancelling GC: %s", GCCause::to_string(cause)); log_info(gc)("%s", msg.buffer()); Events::log(Thread::current(), "%s", msg.buffer()); + _cancel_requested_time = os::elapsedTime(); } } @@ -1790,6 +1766,9 @@ uint ShenandoahHeap::max_workers() { void ShenandoahHeap::stop() { // The shutdown sequence should be able to terminate when GC is running. + // Step 0a. Stop requesting collections. + regulator_thread()->stop(); + // Step 0. Notify policy to disable event recording. _shenandoah_policy->record_shutdown(); @@ -1899,10 +1878,6 @@ address ShenandoahHeap::in_cset_fast_test_addr() { return (address) heap->collection_set()->biased_map_address(); } -address ShenandoahHeap::cancelled_gc_addr() { - return (address) ShenandoahHeap::heap()->_cancelled_gc.addr_of(); -} - address ShenandoahHeap::gc_state_addr() { return (address) ShenandoahHeap::heap()->_gc_state.addr_of(); } @@ -1978,8 +1953,10 @@ void ShenandoahHeap::sync_pinned_region_status() { void ShenandoahHeap::assert_pinned_region_status() { for (size_t i = 0; i < num_regions(); i++) { ShenandoahHeapRegion* r = get_region(i); - assert((r->is_pinned() && r->pin_count() > 0) || (!r->is_pinned() && r->pin_count() == 0), - "Region " SIZE_FORMAT " pinning status is inconsistent", i); + if (active_generation()->contains(r)) { + assert((r->is_pinned() && r->pin_count() > 0) || (!r->is_pinned() && r->pin_count() == 0), + "Region " SIZE_FORMAT " pinning status is inconsistent", i); + } } } #endif @@ -2062,64 +2039,54 @@ class ShenandoahUpdateHeapRefsTask : public AbstractGangTask { void do_work() { T cl; ShenandoahHeapRegion* r = _regions->next(); - ShenandoahMarkingContext* const ctx = _heap->complete_marking_context(); + // We update references for global and young collections. + assert(_heap->active_generation()->is_mark_complete(), "Expected complete marking"); + ShenandoahMarkingContext* const ctx = _heap->marking_context(); while (r != NULL) { HeapWord* update_watermark = r->get_update_watermark(); assert (update_watermark >= r->bottom(), "sanity"); - // Eventually, scanning of old-gen memory regions for the purpose of updating references can happen - // concurrently. This can be done during concurrent evacuation of roots for example. - if (r->is_active() && (r->affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) && !r->is_cset()) { - - // Note that we use this code even if we are doing an old-gen collection and we have a bitmap to - // represent marked objects within the heap region. - // - // It is necessary to process all objects rather than just the marked objects during update-refs of - // an old-gen region as part of an old-gen collection. Otherwise, a subseqent update-refs scan of - // the same region will see stale pointers and crash. - // - // r->top() represents the upper end of memory that has been allocated within this region. - // As new objects are allocated, the value of r->top() increases to accomodate each new - // object. - // At the start of evacuation, "update_watermark" is initalized to represent the value of top(). - // Objects newly allocated during evacuation do not need to be visited during update-refs - // because the to-space invariant which is in force throughout evacuation assures that no from-space - // pointer is written to any newly allocated object. In the case that survivor objects are evacuated - // into this region during evacuation, the region's watermark is incremented to represent the end of - // the memory range known to hold newly evacuated objects. Regions that receive evacuated objects - // are distinct from regions that serve new object allocation requests. A region's watermark is not - // increased when objects are newly allocated within that region during evacuation. - - HeapWord *p = r->bottom(); - ShenandoahObjectToOopBoundedClosure objs(&cl, p, update_watermark); - - // TODO: This code assumes every object ever allocated within this old-gen region is still live. If we - // allow a sweep phase to turn garbage objects into free memory regions, we need to modify this code to - // skip over and/or synchronize access to these free memory regions. There might be races, for example, - // if we are trying to scan one of these free memory regions while a different thread is trying to - // allocate from within a free region. - // - // Alternative approaches are also under consideration. For example: - // 1. Coalesce, fill, and register each range of contiguous dead objects so that subsequent updating of - // references can be done more efficiently. - // 2. Retain the mark bitmap from the most recently completed old GC effort and use this bitmap to allow - // skipping over objects that were not live as of the most recently completed old-gen GC effort. - - // Anything beyond update_watermark does not need to be updated. - while (p < update_watermark) { - oop obj = oop(p); - - // The invocation of do_object() is "borrowed" from the implementation of - // ShenandoahHeap::marked_object_iterate(), which is called by _heap->marked_object_oop_iterate(). - objs.do_object(obj); - p += obj->size(); - + if (r->is_active() && !r->is_cset()) { + if (r->affiliation() == YOUNG_GENERATION || !_heap->mode()->is_generational()) { + _heap->marked_object_oop_iterate(r, &cl, update_watermark); + } else { + assert(r->affiliation() == OLD_GENERATION, "Should not be updating references on FREE regions"); + if (!_heap->is_gc_generation_young()) { + // Old region in a global cycle. + // We need to make sure that the next cycle does not iterate over dead objects + // which haven't had their references updated. + r->oop_iterate(&cl, /*fill_dead_objects*/ true, /* reregister_coalesced_objects */ true); + } else if (ShenandoahBarrierSet::barrier_set()->card_table()->is_dirty(MemRegion(r->bottom(), r->top()))) { + // Old region in a young cycle. + if (r->is_humongous()) { + r->oop_iterate_humongous(&cl); + } else { + // We don't have liveness information about this region. + // Therefore we process all objects, rather than just marked ones. + // Otherwise subsequent traversals will encounter stale pointers. + + // HEY! kelvin thinks we don't have to update refs throughout the entire region r. We only need + // to update refs for objects that span dirty cards. The code in process clusters does that. We cannot invoke process_clusters + // because that's designed to process transitive closure of live objects. Here, we are just looking + // one level deep in each of the relevant regions. But we can copy and paste some of the code from + // there. + + // HEY moreover! Need to figure out how regions are partitioned between worker threads. Is it possible + // that each region is being processed redundantly by each worker thread? + + HeapWord *p = r->bottom(); + ShenandoahObjectToOopBoundedClosure objs(&cl, p, update_watermark); + // Anything beyond update_watermark is not yet allocated or initialized. + while (p < update_watermark) { + oop obj = oop(p); + objs.do_object(obj); + p += obj->size(); + } + } + } } } - else if (r->is_active() && !r->is_cset()) { - _heap->marked_object_oop_iterate(r, &cl, update_watermark); - } if (ShenandoahPacing) { _heap->pacer()->report_updaterefs(pointer_delta(update_watermark, r->bottom())); } @@ -2203,6 +2170,13 @@ void ShenandoahHeap::rebuild_free_set(bool concurrent) { ShenandoahHeapLocker locker(lock()); _free_set->rebuild(); } + + // HEY! this code and rebuild free set used to be in op_final_updaterefs + if (mode()->is_generational() && is_gc_generation_young() && ShenandoahPromoteTenuredRegions) { + ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_promote_tenured_regions); + ShenandoahHeapLocker locker(lock()); + young_generation()->promote_tenured_regions(); + } } void ShenandoahHeap::print_extended_on(outputStream *st) const { @@ -2389,3 +2363,28 @@ void ShenandoahHeap::flush_liveness_cache(uint worker_id) { } } } + +void ShenandoahHeap::purge_old_satb_buffers(bool abandon) { + ((ShenandoahOldGeneration*)_old_generation)->purge_satb_buffers(abandon); +} + +template<> +void ShenandoahGenerationRegionClosure::heap_region_do(ShenandoahHeapRegion* region) { + // Visit young and free regions + if (region->affiliation() != OLD_GENERATION) { + _cl->heap_region_do(region); + } +} + +template<> +void ShenandoahGenerationRegionClosure::heap_region_do(ShenandoahHeapRegion* region) { + // Visit old and free regions + if (region->affiliation() != YOUNG_GENERATION) { + _cl->heap_region_do(region); + } +} + +template<> +void ShenandoahGenerationRegionClosure::heap_region_do(ShenandoahHeapRegion* region) { + _cl->heap_region_do(region); +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index c60e0df025970..5194432e20366 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -28,6 +28,7 @@ #include "gc/shared/markBitMap.hpp" #include "gc/shared/softRefPolicy.hpp" #include "gc/shared/collectedHeap.hpp" +#include "gc/shenandoah/mode/shenandoahGenerationalMode.hpp" #include "gc/shenandoah/shenandoahAsserts.hpp" #include "gc/shenandoah/shenandoahAllocRequest.hpp" #include "gc/shenandoah/shenandoahLock.hpp" @@ -45,12 +46,13 @@ class ConcurrentGCTimer; class ObjectIterateScanRootClosure; class ShenandoahCollectorPolicy; class ShenandoahControlThread; +class ShenandoahRegulatorThread; class ShenandoahGCSession; class ShenandoahGCStateResetter; class ShenandoahGeneration; +class ShenandoahYoungGeneration; class ShenandoahHeuristics; class ShenandoahMarkingContext; -class ShenandoahMode; class ShenandoahPhaseTimings; class ShenandoahHeap; class ShenandoahHeapRegion; @@ -109,6 +111,16 @@ class ShenandoahHeapRegionClosure : public StackObj { virtual bool is_thread_safe() { return false; } }; +template +class ShenandoahGenerationRegionClosure : public ShenandoahHeapRegionClosure { + public: + explicit ShenandoahGenerationRegionClosure(ShenandoahHeapRegionClosure* cl) : _cl(cl) {} + void heap_region_do(ShenandoahHeapRegion* r); + virtual bool is_thread_safe() { return _cl->is_thread_safe(); } + private: + ShenandoahHeapRegionClosure* _cl; +}; + typedef ShenandoahLock ShenandoahHeapLock; typedef ShenandoahLocker ShenandoahHeapLocker; typedef Stack ShenandoahScanObjectStack; @@ -133,12 +145,24 @@ class ShenandoahHeap : public CollectedHeap { // private: ShenandoahHeapLock _lock; + ShenandoahGeneration* _gc_generation; public: ShenandoahHeapLock* lock() { return &_lock; } + ShenandoahGeneration* active_generation() { + // last or latest generation might be a better name here. + return _gc_generation; + } + + void set_gc_generation(ShenandoahGeneration* generation) { + _gc_generation = generation; + } + + bool is_gc_generation_young() const; + // ---------- Initialization, termination, identification, printing routines // public: @@ -151,6 +175,7 @@ class ShenandoahHeap : public CollectedHeap { jint initialize(); void post_initialize(); void initialize_heuristics(); + void initialize_generations(); void initialize_serviceability(); @@ -176,6 +201,8 @@ class ShenandoahHeap : public CollectedHeap { volatile size_t _bytes_allocated_since_gc_start; shenandoah_padding(1); + static size_t young_generation_capacity(size_t total_capacity); + public: void increase_used(size_t bytes); void decrease_used(size_t bytes); @@ -252,22 +279,26 @@ class ShenandoahHeap : public CollectedHeap { // Heap has forwarded objects: needs LRB barriers. HAS_FORWARDED_BITPOS = 0, - // Heap is under marking: needs SATB barriers. - MARKING_BITPOS = 1, + // Young regions are under marking: needs SATB barriers. + YOUNG_MARKING_BITPOS = 1, // Heap is under evacuation: needs LRB barriers. (Set together with HAS_FORWARDED) EVACUATION_BITPOS = 2, // Heap is under updating: needs no additional barriers. UPDATEREFS_BITPOS = 3, + + // Old regions are under marking, still need SATB barriers. + OLD_MARKING_BITPOS = 4 }; enum GCState { STABLE = 0, HAS_FORWARDED = 1 << HAS_FORWARDED_BITPOS, - MARKING = 1 << MARKING_BITPOS, + YOUNG_MARKING = 1 << YOUNG_MARKING_BITPOS, EVACUATION = 1 << EVACUATION_BITPOS, UPDATEREFS = 1 << UPDATEREFS_BITPOS, + OLD_MARKING = 1 << OLD_MARKING_BITPOS }; private: @@ -286,7 +317,8 @@ class ShenandoahHeap : public CollectedHeap { char gc_state() const; static address gc_state_addr(); - void set_concurrent_mark_in_progress(bool in_progress); + void set_concurrent_young_mark_in_progress(bool in_progress); + void set_concurrent_old_mark_in_progress(bool in_progress); void set_evacuation_in_progress(bool in_progress); void set_update_refs_in_progress(bool in_progress); void set_degenerated_gc_in_progress(bool in_progress); @@ -299,6 +331,8 @@ class ShenandoahHeap : public CollectedHeap { inline bool is_stable() const; inline bool is_idle() const; inline bool is_concurrent_mark_in_progress() const; + inline bool is_concurrent_young_mark_in_progress() const; + inline bool is_concurrent_old_mark_in_progress() const; inline bool is_update_refs_in_progress() const; inline bool is_evacuation_in_progress() const; inline bool is_degenerated_gc_in_progress() const; @@ -311,6 +345,8 @@ class ShenandoahHeap : public CollectedHeap { inline bool is_concurrent_weak_root_in_progress() const; private: + void manage_satb_barrier(bool active); + enum CancelState { // Normal state. GC has not been cancelled and is open for cancellation. // Worker threads can suspend for safepoint. @@ -326,17 +362,17 @@ class ShenandoahHeap : public CollectedHeap { NOT_CANCELLED }; + double _cancel_requested_time; ShenandoahSharedEnumFlag _cancelled_gc; bool try_cancel_gc(); public: - static address cancelled_gc_addr(); - inline bool cancelled_gc() const; inline bool check_cancelled_gc_and_yield(bool sts_active = true); - inline void clear_cancelled_gc(); + inline void clear_cancelled_gc(bool clear_oom_handler = true); + void cancel_concurrent_mark(); void cancel_gc(GCCause::Cause cause); public: @@ -346,11 +382,7 @@ class ShenandoahHeap : public CollectedHeap { private: // GC support - // Reset bitmap, prepare regions for new GC cycle - void prepare_gc(); - void prepare_regions_and_collection_set(bool concurrent); // Evacuation - void prepare_evacuation(bool concurrent); void evacuate_collection_set(bool concurrent); // Concurrent root processing void prepare_concurrent_roots(); @@ -374,12 +406,14 @@ class ShenandoahHeap : public CollectedHeap { // // Mark support private: - ShenandoahGeneration* _young_generation; + ShenandoahYoungGeneration* _young_generation; ShenandoahGeneration* _global_generation; + ShenandoahGeneration* _old_generation; + ShenandoahControlThread* _control_thread; + ShenandoahRegulatorThread* _regulator_thread; ShenandoahCollectorPolicy* _shenandoah_policy; ShenandoahMode* _gc_mode; - ShenandoahHeuristics* _heuristics; ShenandoahFreeSet* _free_set; ShenandoahPacer* _pacer; ShenandoahVerifier* _verifier; @@ -387,13 +421,15 @@ class ShenandoahHeap : public CollectedHeap { ShenandoahPhaseTimings* _phase_timings; ShenandoahControlThread* control_thread() { return _control_thread; } + ShenandoahRegulatorThread* regulator_thread() { return _regulator_thread; } public: - ShenandoahGeneration* young_generation() const { return _young_generation; } + ShenandoahYoungGeneration* young_generation() const { return _young_generation; } ShenandoahGeneration* global_generation() const { return _global_generation; } + ShenandoahGeneration* old_generation() const { return _old_generation; } + ShenandoahCollectorPolicy* shenandoah_policy() const { return _shenandoah_policy; } ShenandoahMode* mode() const { return _gc_mode; } - ShenandoahHeuristics* heuristics() const { return _heuristics; } ShenandoahFreeSet* free_set() const { return _free_set; } ShenandoahPacer* pacer() const { return _pacer; } @@ -468,6 +504,8 @@ class ShenandoahHeap : public CollectedHeap { bool is_in(const void* p) const; bool is_in_young(const void* p) const; + bool is_in_old(const void* p) const; + inline bool is_old(oop pobj) const; MemRegion reserved_region() const { return _reserved; } bool is_in_reserved(const void* addr) const { return _reserved.contains(addr); } @@ -572,8 +610,6 @@ class ShenandoahHeap : public CollectedHeap { public: inline ShenandoahMarkingContext* complete_marking_context() const; inline ShenandoahMarkingContext* marking_context() const; - inline void mark_complete_marking_context(); - inline void mark_incomplete_marking_context(); template inline void marked_object_iterate(ShenandoahHeapRegion* region, T* cl); @@ -584,8 +620,6 @@ class ShenandoahHeap : public CollectedHeap { template inline void marked_object_oop_iterate(ShenandoahHeapRegion* region, T* cl, HeapWord* limit); - void reset_mark_bitmap(); - // SATB barriers hooks inline bool requires_marking(const void* entry) const; @@ -606,6 +640,8 @@ class ShenandoahHeap : public CollectedHeap { ShenandoahCollectionSet* _collection_set; ShenandoahEvacOOMHandler _oom_evac_handler; + inline oop try_evacuate_object(oop src, Thread* thread, ShenandoahHeapRegion* from_region, ShenandoahRegionAffiliation target_gen); + public: static address in_cset_fast_test_addr(); @@ -617,7 +653,7 @@ class ShenandoahHeap : public CollectedHeap { // Checks if location is in the collection set. Can be interior pointer, not the oop itself. inline bool in_collection_set_loc(void* loc) const; - // Evacuates object src. Returns the evacuated object, either evacuated + // Evacuates or promotes object src. Returns the evacuated object, either evacuated // by this thread, or by some other thread. inline oop evacuate_object(oop src, Thread* thread); @@ -632,7 +668,7 @@ class ShenandoahHeap : public CollectedHeap { public: inline RememberedScanner* card_scan() { return _card_scan; } - + void clear_cards_for(ShenandoahHeapRegion* region); // ---------- Helper functions // @@ -651,6 +687,9 @@ class ShenandoahHeap : public CollectedHeap { void deduplicate_string(oop str); + static inline void increase_object_age(oop obj, uint additional_age); + + void purge_old_satb_buffers(bool abandon); private: void trash_cset_regions(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp index 60e8772ac2163..647352f3f48c5 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp @@ -41,6 +41,7 @@ #include "gc/shenandoah/shenandoahControlThread.hpp" #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp" #include "gc/shenandoah/shenandoahThreadLocalData.hpp" +#include "gc/shenandoah/mode/shenandoahMode.hpp" #include "oops/compressedOops.inline.hpp" #include "oops/oop.inline.hpp" #include "runtime/atomic.hpp" @@ -182,9 +183,17 @@ inline bool ShenandoahHeap::check_cancelled_gc_and_yield(bool sts_active) { } } -inline void ShenandoahHeap::clear_cancelled_gc() { +inline void ShenandoahHeap::clear_cancelled_gc(bool clear_oom_handler) { _cancelled_gc.set(CANCELLABLE); - _oom_evac_handler.clear(); + if (_cancel_requested_time > 0) { + double cancel_time = os::elapsedTime() - _cancel_requested_time; + log_info(gc)("GC cancellation took %.3fs", cancel_time); + _cancel_requested_time = 0; + } + + if (clear_oom_handler) { + _oom_evac_handler.clear(); + } } inline HeapWord* ShenandoahHeap::allocate_from_gclab(Thread* thread, size_t size) { @@ -214,18 +223,38 @@ inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) { assert(ShenandoahThreadLocalData::is_evac_allowed(thread), "must be enclosed in oom-evac scope"); - ShenandoahRegionAffiliation target_gen = heap_region_containing(p)->affiliation(); - if (target_gen == YOUNG_GENERATION) { - if (ShenandoahForwarding::is_forwarded(p)) { + ShenandoahHeapRegion* r = heap_region_containing(p); + assert(!r->is_humongous(), "never evacuate humongous objects"); + + ShenandoahRegionAffiliation target_gen = r->affiliation(); + if (mode()->is_generational() && ShenandoahHeap::heap()->is_gc_generation_young() && + target_gen == YOUNG_GENERATION && ShenandoahPromoteTenuredObjects) { + markWord mark = p->mark(); + if (mark.is_marked()) { + // Already forwarded. return ShenandoahBarrierSet::resolve_forwarded(p); - } else if (p->mark().age() > InitialTenuringThreshold) { - //tty->print_cr("promoting object: " PTR_FORMAT, p2i(p)); - //target_gen = OLD_GEN; + } + if (mark.has_displaced_mark_helper()) { + // We don't want to deal with MT here just to ensure we read the right mark word. + // Skip the potential promotion attempt for this one. + } else if (mark.age() >= InitialTenuringThreshold) { + oop result = try_evacuate_object(p, thread, r, OLD_GENERATION); + if (result != NULL) { + // TODO: Just marking the cards covering this object dirty + // may overall be less efficient than scanning it now for references to young gen + // or other alternatives like deferred card marking or scanning. + // We should revisit this. + // Furthermore, the object start should be registered for remset scanning. + MemRegion mr(cast_from_oop(result), result->size()); + ShenandoahBarrierSet::barrier_set()->card_table()->invalidate(mr); + return result; + } } } + return try_evacuate_object(p, thread, r, target_gen); +} - assert(!heap_region_containing(p)->is_humongous(), "never evacuate humongous objects"); - +inline oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region, ShenandoahRegionAffiliation target_gen) { bool alloc_from_gclab = true; HeapWord* copy = NULL; size_t size = p->size(); @@ -249,6 +278,11 @@ inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) { #endif if (copy == NULL) { + if (target_gen == OLD_GENERATION && from_region->affiliation() == YOUNG_GENERATION) { + // Indicate that a promotion attempt failed. + return NULL; + } + control_thread()->handle_alloc_failure_evac(size); _oom_evac_handler.handle_out_of_memory_during_evacuation(); @@ -259,24 +293,25 @@ inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) { // Copy the object: Copy::aligned_disjoint_words(cast_from_oop(p), copy, size); - // Try to install the new forwarding pointer. oop copy_val = oop(copy); + if (target_gen == YOUNG_GENERATION) { + // Increment the age in young copies, absorbing region age. + // (Only retired regions will have more than zero age to pass along.) + + ShenandoahHeap::increase_object_age(copy_val, from_region->age() + 1); + + // Note that p may have been forwarded by another thread, + // anywhere between here and the check above for forwarding. + // In that case try_update_forwardee() below will not be successful + // and the increment we just performed will simply be forgotten, + // but it will have succeeded in said other thread. + } + + // Try to install the new forwarding pointer. oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val); if (result == copy_val) { // Successfully evacuated. Our copy is now the public one! shenandoah_assert_correct(NULL, copy_val); - - // Hey! This code showed up in a merge conflict. It has "nothing" to do with the patch that - // was merged, so kdnilsen is leaving it in place as is. However, it looks to me like the object's - // age should be incremented before the copy is committed to avoid the need for synchronization here. - // - // kdnilsen believes the following code is replaced/relocated in a subsequent commit. - - // Increment age in young copies. - if (target_gen == YOUNG_GENERATION) { - copy_val->incr_age(); - } - return copy_val; } else { // Failed to evacuate. We need to deal with the object that is left behind. Since this @@ -301,6 +336,21 @@ inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) { } } +void ShenandoahHeap::increase_object_age(oop obj, uint additional_age) { + markWord w = obj->has_displaced_mark() ? obj->displaced_mark() : obj->mark(); + w = w.set_age(MIN2(markWord::max_age, w.age() + additional_age)); + if (obj->has_displaced_mark()) { + obj->set_displaced_mark(w); + } else { + obj->set_mark(w); + } +} + + +inline bool ShenandoahHeap::is_old(oop obj) const { + return is_gc_generation_young() && is_in_old(obj); +} + inline bool ShenandoahHeap::requires_marking(const void* entry) const { oop obj = oop(entry); return !_marking_context->is_marked_strong(obj); @@ -321,11 +371,19 @@ inline bool ShenandoahHeap::is_stable() const { } inline bool ShenandoahHeap::is_idle() const { - return _gc_state.is_unset(MARKING | EVACUATION | UPDATEREFS); + return _gc_state.is_unset(YOUNG_MARKING | OLD_MARKING | EVACUATION | UPDATEREFS); } inline bool ShenandoahHeap::is_concurrent_mark_in_progress() const { - return _gc_state.is_set(MARKING); + return _gc_state.is_set(YOUNG_MARKING | OLD_MARKING); +} + +inline bool ShenandoahHeap::is_concurrent_young_mark_in_progress() const { + return _gc_state.is_set(YOUNG_MARKING); +} + +inline bool ShenandoahHeap::is_concurrent_old_mark_in_progress() const { + return _gc_state.is_set(OLD_MARKING); } inline bool ShenandoahHeap::is_evacuation_in_progress() const { @@ -373,8 +431,9 @@ template inline void ShenandoahHeap::marked_object_iterate(ShenandoahHeapRegion* region, T* cl, HeapWord* limit) { assert(! region->is_humongous_continuation(), "no humongous continuation regions here"); - ShenandoahMarkingContext* const ctx = complete_marking_context(); - assert(ctx->is_complete(), "sanity"); + ShenandoahMarkingContext* const ctx = marking_context(); + // HEY! All callers (at the time of this writing) have already asserted the mark context is complete. + // assert(ctx->is_complete(), "sanity"); HeapWord* tams = ctx->top_at_mark_start(region); @@ -505,14 +564,6 @@ inline ShenandoahHeapRegion* const ShenandoahHeap::get_region(size_t region_idx) } } -inline void ShenandoahHeap::mark_complete_marking_context() { - _marking_context->mark_complete(); -} - -inline void ShenandoahHeap::mark_incomplete_marking_context() { - _marking_context->mark_incomplete(); -} - inline ShenandoahMarkingContext* ShenandoahHeap::complete_marking_context() const { assert (_marking_context->is_complete()," sanity"); return _marking_context; @@ -522,4 +573,10 @@ inline ShenandoahMarkingContext* ShenandoahHeap::marking_context() const { return _marking_context; } +inline void ShenandoahHeap::clear_cards_for(ShenandoahHeapRegion* region) { + if (mode()->is_generational()) { + _card_scan->mark_range_as_empty(region->bottom(), (uint32_t) (region->end() - region->bottom())); + } +} + #endif // SHARE_GC_SHENANDOAH_SHENANDOAHHEAP_INLINE_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index a0d168e3d311c..6061050f28154 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -25,10 +25,14 @@ #include "precompiled.hpp" #include "gc/shared/space.inline.hpp" #include "gc/shared/tlab_globals.hpp" +#include "gc/shenandoah/shenandoahCardTable.hpp" #include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahHeapRegion.hpp" #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp" +#include "gc/shenandoah/shenandoahGeneration.hpp" +#include "gc/shenandoah/shenandoahYoungGeneration.hpp" +#include "gc/shenandoah/shenandoahScanRemembered.inline.hpp" #include "jfr/jfrEvents.hpp" #include "memory/allocation.hpp" #include "memory/iterator.inline.hpp" @@ -43,6 +47,7 @@ #include "runtime/safepoint.hpp" #include "utilities/powerOfTwo.hpp" + size_t ShenandoahHeapRegion::RegionCount = 0; size_t ShenandoahHeapRegion::RegionSizeBytes = 0; size_t ShenandoahHeapRegion::RegionSizeWords = 0; @@ -68,7 +73,8 @@ ShenandoahHeapRegion::ShenandoahHeapRegion(HeapWord* start, size_t index, bool c _live_data(0), _critical_pins(0), _update_watermark(start), - _affiliation(ShenandoahRegionAffiliation::FREE) { + _affiliation(ShenandoahRegionAffiliation::FREE), + _age(0) { assert(Universe::on_page_boundary(_bottom) && Universe::on_page_boundary(_end), "invalid space boundaries"); @@ -87,7 +93,7 @@ void ShenandoahHeapRegion::report_illegal_transition(const char *method) { void ShenandoahHeapRegion::make_regular_allocation() { shenandoah_assert_heaplocked(); - + reset_age(); switch (_state) { case _empty_uncommitted: do_commit(); @@ -105,7 +111,7 @@ void ShenandoahHeapRegion::make_regular_bypass() { shenandoah_assert_heaplocked(); assert (ShenandoahHeap::heap()->is_full_gc_in_progress() || ShenandoahHeap::heap()->is_degenerated_gc_in_progress(), "only for full or degen GC"); - + reset_age(); switch (_state) { case _empty_uncommitted: do_commit(); @@ -128,6 +134,7 @@ void ShenandoahHeapRegion::make_regular_bypass() { void ShenandoahHeapRegion::make_humongous_start() { shenandoah_assert_heaplocked(); + reset_age(); switch (_state) { case _empty_uncommitted: do_commit(); @@ -142,7 +149,7 @@ void ShenandoahHeapRegion::make_humongous_start() { void ShenandoahHeapRegion::make_humongous_start_bypass() { shenandoah_assert_heaplocked(); assert (ShenandoahHeap::heap()->is_full_gc_in_progress(), "only for full GC"); - + reset_age(); switch (_state) { case _empty_committed: case _regular: @@ -157,6 +164,7 @@ void ShenandoahHeapRegion::make_humongous_start_bypass() { void ShenandoahHeapRegion::make_humongous_cont() { shenandoah_assert_heaplocked(); + reset_age(); switch (_state) { case _empty_uncommitted: do_commit(); @@ -171,7 +179,7 @@ void ShenandoahHeapRegion::make_humongous_cont() { void ShenandoahHeapRegion::make_humongous_cont_bypass() { shenandoah_assert_heaplocked(); assert (ShenandoahHeap::heap()->is_full_gc_in_progress(), "only for full GC"); - + reset_age(); switch (_state) { case _empty_committed: case _regular: @@ -230,6 +238,7 @@ void ShenandoahHeapRegion::make_unpinned() { void ShenandoahHeapRegion::make_cset() { shenandoah_assert_heaplocked(); + reset_age(); switch (_state) { case _regular: set_state(_cset); @@ -242,6 +251,7 @@ void ShenandoahHeapRegion::make_cset() { void ShenandoahHeapRegion::make_trash() { shenandoah_assert_heaplocked(); + reset_age(); switch (_state) { case _cset: // Reclaiming cset regions @@ -262,11 +272,13 @@ void ShenandoahHeapRegion::make_trash_immediate() { // On this path, we know there are no marked objects in the region, // tell marking context about it to bypass bitmap resets. - ShenandoahHeap::heap()->complete_marking_context()->reset_top_bitmap(this); + assert(ShenandoahHeap::heap()->active_generation()->is_mark_complete(), "Marking should be complete here."); + ShenandoahHeap::heap()->marking_context()->reset_top_bitmap(this); } void ShenandoahHeapRegion::make_empty() { shenandoah_assert_heaplocked(); + reset_age(); switch (_state) { case _trash: set_state(_empty_committed); @@ -364,13 +376,13 @@ void ShenandoahHeapRegion::print_on(outputStream* st) const { ShouldNotReachHere(); } switch (_affiliation) { - case FREE: + case ShenandoahRegionAffiliation::FREE: st->print("|F"); break; - case YOUNG_GENERATION: + case ShenandoahRegionAffiliation::YOUNG_GENERATION: st->print("|Y"); break; - case OLD_GENERATION: + case ShenandoahRegionAffiliation::OLD_GENERATION: st->print("|O"); break; default: @@ -391,23 +403,71 @@ void ShenandoahHeapRegion::print_on(outputStream* st) const { st->cr(); } -void ShenandoahHeapRegion::oop_iterate(OopIterateClosure* blk) { +void ShenandoahHeapRegion::oop_iterate(OopIterateClosure* blk, bool fill_dead_objects, bool reregister_coalesced_objects) { if (!is_active()) return; if (is_humongous()) { + if (fill_dead_objects && !reregister_coalesced_objects) { + ShenandoahHeap::heap()->card_scan()->register_object(bottom()); + } oop_iterate_humongous(blk); } else { - oop_iterate_objects(blk); + oop_iterate_objects(blk, fill_dead_objects, reregister_coalesced_objects); } } -void ShenandoahHeapRegion::oop_iterate_objects(OopIterateClosure* blk) { - assert(! is_humongous(), "no humongous region here"); +void ShenandoahHeapRegion::oop_iterate_objects(OopIterateClosure* blk, bool fill_dead_objects, bool reregister_coalesced_objects) { + assert(!is_humongous(), "no humongous region here"); HeapWord* obj_addr = bottom(); HeapWord* t = top(); - // Could call objects iterate, but this is easier. - while (obj_addr < t) { - oop obj = oop(obj_addr); - obj_addr += obj->oop_iterate_size(blk); + + if (!fill_dead_objects) { + while (obj_addr < t) { + oop obj = oop(obj_addr); + assert(obj->klass() != NULL, "klass should not be NULL"); + obj_addr += obj->oop_iterate_size(blk); + } + } else { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + ShenandoahMarkingContext* marking_context = heap->marking_context(); + assert(heap->active_generation()->is_mark_complete(), "sanity"); + + HeapWord* fill_addr = NULL; + size_t fill_size = 0; + while (obj_addr < t) { + oop obj = oop(obj_addr); + if (marking_context->is_marked(obj)) { + if (fill_addr != NULL) { + if (reregister_coalesced_objects) { // change existing crossing map information + heap->card_scan()->coalesce_objects(fill_addr, fill_size); + } else { // establish new crossing map information + heap->card_scan()->register_object(fill_addr); + } + ShenandoahHeap::fill_with_object(fill_addr, fill_size); + fill_addr = NULL; + } + assert(obj->klass() != NULL, "klass should not be NULL"); + if (!reregister_coalesced_objects) + heap->card_scan()->register_object(obj_addr); + obj_addr += obj->oop_iterate_size(blk); + } else { + int size = obj->size(); + if (fill_addr == NULL) { + fill_addr = obj_addr; + fill_size = size; + } else { + fill_size += size; + } + obj_addr += size; + } + } + if (fill_addr != NULL) { + ShenandoahHeap::fill_with_object(fill_addr, fill_size); + if (reregister_coalesced_objects) { // change existing crossing map information + heap->card_scan()->coalesce_objects(fill_addr, fill_size); + } else { // establish new crossing map information + heap->card_scan()->register_object(fill_addr); + } + } } } @@ -436,16 +496,25 @@ ShenandoahHeapRegion* ShenandoahHeapRegion::humongous_start_region() const { } void ShenandoahHeapRegion::recycle() { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + + if (affiliation() == YOUNG_GENERATION) { + heap->young_generation()->decrease_used(used()); + } else if (affiliation() == OLD_GENERATION) { + heap->old_generation()->decrease_used(used()); + } + set_top(bottom()); clear_live_data(); reset_alloc_metadata(); - ShenandoahHeap::heap()->marking_context()->reset_top_at_mark_start(this); + heap->marking_context()->reset_top_at_mark_start(this); set_update_watermark(bottom()); make_empty(); - _affiliation = ShenandoahRegionAffiliation::FREE; + + set_affiliation(FREE); if (ZapUnusedHeapArea) { SpaceMangler::mangle_region(MemRegion(bottom(), end())); @@ -680,61 +749,38 @@ size_t ShenandoahHeapRegion::pin_count() const { return Atomic::load(&_critical_pins); } -class UpdateOopCardValueClosure : public BasicOopIterateClosure { - CardTable* _card_table; - -public: - UpdateOopCardValueClosure(CardTable *card_table) : _card_table(card_table) { } - - void do_oop(oop* p) { - if (ShenandoahHeap::heap()->is_in_young(*p)) { - volatile CardTable::CardValue* card_value = _card_table->byte_for(*p); - *card_value = CardTable::dirty_card_val(); - } - } - - void do_oop(narrowOop* p) { - ShouldNotReachHere(); +void ShenandoahHeapRegion::set_affiliation(ShenandoahRegionAffiliation new_affiliation) { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + if (_affiliation == new_affiliation) { + return; } -}; -class UpdateObjectCardValuesClosure : public BasicOopIterateClosure { - UpdateOopCardValueClosure* _oop_closure; - -public: - UpdateObjectCardValuesClosure(UpdateOopCardValueClosure *oop_closure) : - _oop_closure(oop_closure) { + if (!heap->mode()->is_generational()) { + _affiliation = new_affiliation; + return; } - void do_oop(oop* p) { - (*p)->oop_iterate(_oop_closure); - } + log_trace(gc)("Changing affiliation of region %zu from %s to %s", + index(), affiliation_name(_affiliation), affiliation_name(new_affiliation)); - void do_oop(narrowOop* p) { - ShouldNotReachHere(); + if (_affiliation == ShenandoahRegionAffiliation::YOUNG_GENERATION) { + heap->young_generation()->decrement_affiliated_region_count(); + } else if (_affiliation == ShenandoahRegionAffiliation::OLD_GENERATION) { + heap->old_generation()->decrement_affiliated_region_count(); } -}; -void ShenandoahHeapRegion::set_affiliation(ShenandoahRegionAffiliation new_affiliation) { - if (_affiliation == new_affiliation) { - return; - } CardTable* card_table = ShenandoahBarrierSet::barrier_set()->card_table(); switch (new_affiliation) { case FREE: + assert(!has_live(), "Free region should not have live data"); card_table->clear_MemRegion(MemRegion(_bottom, _end)); break; case YOUNG_GENERATION: + reset_age(); + heap->young_generation()->increment_affiliated_region_count(); break; case OLD_GENERATION: - if (_affiliation == YOUNG_GENERATION) { - assert(SafepointSynchronize::is_at_safepoint(), "old gen card values must be updated in a safepoint"); - card_table->clear_MemRegion(MemRegion(_bottom, _end)); - - UpdateOopCardValueClosure oop_closure(card_table); - UpdateObjectCardValuesClosure object_closure(&oop_closure); - oop_iterate(&object_closure); - } + heap->old_generation()->increment_affiliated_region_count(); break; default: ShouldNotReachHere(); @@ -742,3 +788,82 @@ void ShenandoahHeapRegion::set_affiliation(ShenandoahRegionAffiliation new_affil } _affiliation = new_affiliation; } + +class UpdateCardValuesClosure : public BasicOopIterateClosure { +private: + void update_card_value(void* address, oop obj) { + if (ShenandoahHeap::heap()->is_in_young(obj)) { + volatile CardTable::CardValue* card_value = ShenandoahBarrierSet::barrier_set()->card_table()->byte_for(address); + *card_value = CardTable::dirty_card_val(); + } + } + +public: + void do_oop(oop* p) { + oop obj = *p; + if (obj != NULL) { + update_card_value(p, obj); + } + } + + void do_oop(narrowOop* p) { + narrowOop o = RawAccess<>::oop_load(p); + if (!CompressedOops::is_null(o)) { + oop obj = CompressedOops::decode_not_null(o); + assert(oopDesc::is_oop(obj), "must be a valid oop"); + update_card_value(p, obj); + } + } +}; + +void ShenandoahHeapRegion::promote() { + assert(SafepointSynchronize::is_at_safepoint(), "must be at a safepoint"); + + ShenandoahHeap* heap = ShenandoahHeap::heap(); + ShenandoahMarkingContext* marking_context = heap->marking_context(); + assert(heap->active_generation()->is_mark_complete(), "sanity"); + assert(affiliation() == YOUNG_GENERATION, "Only young regions can be promoted"); + + heap->young_generation()->decrease_used(used()); + heap->old_generation()->increase_used(used()); + + UpdateCardValuesClosure update_card_values; + if (is_humongous_start()) { + oop obj = oop(bottom()); + assert(marking_context->is_marked(obj), "promoted humongous object should be alive"); + + size_t index_limit = index() + ShenandoahHeapRegion::required_regions(obj->size() * HeapWordSize); + heap->card_scan()->register_object(bottom()); + for (size_t i = index(); i < index_limit; i++) { + ShenandoahHeapRegion* r = heap->get_region(i); + log_debug(gc)("promoting region " SIZE_FORMAT ", clear cards from " SIZE_FORMAT " to " SIZE_FORMAT, + r->index(), (size_t) r->bottom(), (size_t) r->top()); + + ShenandoahBarrierSet::barrier_set()->card_table()->clear_MemRegion(MemRegion(r->bottom(), r->end())); + r->set_affiliation(OLD_GENERATION); + } + // HEY! Better to call ShenandoahHeap::heap()->card_scan()->mark_range_as_clean(r->bottom(), obj->size()) + // and skip the calls to clear_MemRegion() above. + + // Iterate over all humongous regions that are spanned by the humongous object obj. The remnant + // of memory in the last humongous region that is not spanned by obj is currently not used. + obj->oop_iterate(&update_card_values); + } else { + log_debug(gc)("promoting region " SIZE_FORMAT ", clear cards from " SIZE_FORMAT " to " SIZE_FORMAT, + index(), (size_t) bottom(), (size_t) top()); + assert(!is_humongous_continuation(), "should not promote humongous object continuation in isolation"); + + // Rather than scanning entire contents of the promoted region right now to determine which + // cards to mark dirty, we just mark them all as dirty. Later, when we scan the remembered + // set, we will clear cards that are found to not contain live references to young memory. + // Ultimately, this approach is more efficient as it only scans the "dirty" cards once and + // the clean cards once. The alternative approach of scanning all cards now and then scanning + // dirty cards again at next concurrent mark pass scans the clean cards once and the dirty + // cards twice. + + // HEY! Better to call ShenandoahHeap::heap()->card_scan()->mark_range_as_dirty(r->bottom(), obj->size()); + ShenandoahBarrierSet::barrier_set()->card_table()->dirty_MemRegion(MemRegion(bottom(), end())); + set_affiliation(OLD_GENERATION); + oop_iterate_objects(&update_card_values, /*fill_dead_objects*/ true, /* reregister_coalesced_objects */ false); + } +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp index 167f4731ae508..83455eabbd947 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp @@ -198,6 +198,8 @@ class ShenandoahHeapRegion { bool is_committed() const { return !is_empty_uncommitted(); } bool is_cset() const { return _state == _cset || _state == _pinned_cset; } bool is_pinned() const { return _state == _pinned || _state == _pinned_cset || _state == _pinned_humongous_start; } + bool is_young() const { return _affiliation == YOUNG_GENERATION; } + bool is_old() const { return _affiliation == OLD_GENERATION; } // Macro-properties: bool is_alloc_allowed() const { return is_empty() || is_regular() || _state == _pinned; } @@ -247,6 +249,7 @@ class ShenandoahHeapRegion { HeapWord* volatile _update_watermark; ShenandoahRegionAffiliation _affiliation; + uint _age; public: ShenandoahHeapRegion(HeapWord* start, size_t index, bool committed); @@ -357,7 +360,8 @@ class ShenandoahHeapRegion { void recycle(); - void oop_iterate(OopIterateClosure* cl); + void oop_iterate(OopIterateClosure* cl, bool fill_dead_objects = false, bool reregister_coalesced_objects = false); + void oop_iterate_humongous(OopIterateClosure* cl); HeapWord* block_start(const void* p) const; size_t block_size(const HeapWord* p) const; @@ -395,12 +399,17 @@ class ShenandoahHeapRegion { void set_affiliation(ShenandoahRegionAffiliation new_affiliation); + uint age() { return _age; } + void increment_age() { if (_age < markWord::max_age) { _age++; } } + void reset_age() { _age = 0; } + + void promote(); + private: void do_commit(); void do_uncommit(); - void oop_iterate_objects(OopIterateClosure* cl); - void oop_iterate_humongous(OopIterateClosure* cl); + void oop_iterate_objects(OopIterateClosure* cl, bool fill_dead_objects, bool reregister_coalesced_objects); inline void internal_increase_live_data(size_t s); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp index 2317bbe2059e0..dfdd95d15799d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp @@ -23,6 +23,8 @@ */ #include "precompiled.hpp" + +#include "gc/shenandoah/shenandoahGeneration.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" #include "gc/shenandoah/shenandoahHeapRegionSet.hpp" @@ -82,9 +84,17 @@ void ShenandoahHeapRegionCounters::update() { ShenandoahHeap* heap = ShenandoahHeap::heap(); jlong status = 0; - if (heap->is_concurrent_mark_in_progress()) status |= 1 << 0; - if (heap->is_evacuation_in_progress()) status |= 1 << 1; - if (heap->is_update_refs_in_progress()) status |= 1 << 2; + + if (heap->is_concurrent_young_mark_in_progress()) status |= 1 << 0; + if (heap->is_concurrent_old_mark_in_progress()) status |= 1 << 1; + if (heap->is_evacuation_in_progress()) status |= 1 << 2; + if (heap->is_update_refs_in_progress()) status |= 1 << 3; + if (heap->is_degenerated_gc_in_progress()) status |= 1 << 4; + if (heap->is_full_gc_in_progress()) status |= 1 << 5; + if (heap->active_generation() != NULL) { + status |= (heap->active_generation()->generation_mode() << 6); + } + _status->set_value(status); _timestamp->set_value(os::elapsed_counter()); @@ -102,11 +112,13 @@ void ShenandoahHeapRegionCounters::update() { data |= ((100 * r->get_tlab_allocs() / rs) & PERCENT_MASK) << TLAB_SHIFT; data |= ((100 * r->get_gclab_allocs() / rs) & PERCENT_MASK) << GCLAB_SHIFT; data |= ((100 * r->get_shared_allocs() / rs) & PERCENT_MASK) << SHARED_SHIFT; + + data |= (r->age() & AGE_MASK) << AGE_SHIFT; + data |= (r->affiliation() & AFFILIATION_MASK) << AFFILIATION_SHIFT; data |= (r->state_ordinal() & STATUS_MASK) << STATUS_SHIFT; _regions_data[i]->set_value(data); } } - } } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.hpp index f0d4c2ad38f70..5400585434a1b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.hpp @@ -53,22 +53,26 @@ * - bits 28-34 shared allocated memory in percent * - bits 35-41 * - bits 42-50 - * - bits 51-57 + * - bits 51-55 age + * - bits 56-57 affiliation: 0 = free, young = 1, old = 2 * - bits 58-63 status * - bits describe the state as recorded in ShenandoahHeapRegion */ class ShenandoahHeapRegionCounters : public CHeapObj { private: - static const jlong PERCENT_MASK = 0x7f; - static const jlong STATUS_MASK = 0x3f; + static const jlong PERCENT_MASK = 0x7f; + static const jlong AGE_MASK = 0x1f; + static const jlong AFFILIATION_MASK = 0x03; + static const jlong STATUS_MASK = 0x3f; - static const jlong USED_SHIFT = 0; - static const jlong LIVE_SHIFT = 7; - static const jlong TLAB_SHIFT = 14; - static const jlong GCLAB_SHIFT = 21; - static const jlong SHARED_SHIFT = 28; - - static const jlong STATUS_SHIFT = 58; + static const jlong USED_SHIFT = 0; + static const jlong LIVE_SHIFT = 7; + static const jlong TLAB_SHIFT = 14; + static const jlong GCLAB_SHIFT = 21; + static const jlong SHARED_SHIFT = 28; + static const jlong AGE_SHIFT = 51; + static const jlong AFFILIATION_SHIFT = 56; + static const jlong STATUS_SHIFT = 58; char* _name_space; PerfLongVariable** _regions_data; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahInitLogger.cpp b/src/hotspot/share/gc/shenandoah/shenandoahInitLogger.cpp index 9f0233dd08c14..27ea64f587778 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahInitLogger.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahInitLogger.cpp @@ -26,6 +26,7 @@ #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahHeapRegion.hpp" #include "gc/shenandoah/shenandoahInitLogger.hpp" +#include "gc/shenandoah/shenandoahGeneration.hpp" #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" #include "gc/shenandoah/mode/shenandoahMode.hpp" #include "logging/log.hpp" @@ -41,7 +42,7 @@ void ShenandoahInitLogger::print_heap() { heap->mode()->name()); log_info(gc, init)("Heuristics: %s", - heap->heuristics()->name()); + heap->global_generation()->heuristics()->name()); log_info(gc, init)("Heap Region Count: " SIZE_FORMAT, ShenandoahHeapRegion::region_count()); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp index 35d4aa739573e..e6300b00e79cc 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp @@ -27,6 +27,7 @@ #include "gc/shenandoah/shenandoahBarrierSet.hpp" #include "gc/shenandoah/shenandoahClosures.inline.hpp" +#include "gc/shenandoah/shenandoahGeneration.hpp" #include "gc/shenandoah/shenandoahMark.inline.hpp" #include "gc/shenandoah/shenandoahOopClosures.inline.hpp" #include "gc/shenandoah/shenandoahReferenceProcessor.hpp" @@ -34,30 +35,25 @@ #include "gc/shenandoah/shenandoahUtils.hpp" #include "gc/shenandoah/shenandoahVerifier.hpp" -ShenandoahMarkRefsSuperClosure::ShenandoahMarkRefsSuperClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) : +ShenandoahMarkRefsSuperClosure::ShenandoahMarkRefsSuperClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp, ShenandoahObjToScanQueue* old_q) : MetadataVisitingOopIterateClosure(rp), _queue(q), + _old_queue(old_q), _mark_context(ShenandoahHeap::heap()->marking_context()), _weak(false) { } -ShenandoahMark::ShenandoahMark() : - _task_queues(ShenandoahHeap::heap()->marking_context()->task_queues()) { -} - -void ShenandoahMark::clear() { - // Clean up marking stacks. - ShenandoahObjToScanQueueSet* queues = ShenandoahHeap::heap()->marking_context()->task_queues(); - queues->clear(); - - // Cancel SATB buffers. - ShenandoahBarrierSet::satb_mark_queue_set().abandon_partial_marking(); +ShenandoahMark::ShenandoahMark(ShenandoahGeneration* generation) : + _generation(generation), + _task_queues(generation->task_queues()), + _old_gen_task_queues(generation->old_gen_task_queues()) { } template void ShenandoahMark::mark_loop_prework(uint w, TaskTerminator *t, ShenandoahReferenceProcessor *rp, - bool strdedup) { + bool strdedup, bool update_refs) { ShenandoahObjToScanQueue* q = get_queue(w); + ShenandoahObjToScanQueue* old = get_old_queue(w); ShenandoahHeap* const heap = ShenandoahHeap::heap(); ShenandoahLiveData* ld = heap->get_liveness_cache(w); @@ -65,38 +61,38 @@ void ShenandoahMark::mark_loop_prework(uint w, TaskTerminator *t, ShenandoahRefe // TODO: We can clean up this if we figure out how to do templated oop closures that // play nice with specialized_oop_iterators. if (heap->unload_classes()) { - if (heap->has_forwarded_objects()) { + if (update_refs) { if (strdedup) { - ShenandoahMarkUpdateRefsMetadataDedupClosure cl(q, rp); + ShenandoahMarkUpdateRefsMetadataDedupClosure cl(q, rp, old); mark_loop_work, GENERATION, CANCELLABLE>(&cl, ld, w, t); } else { - ShenandoahMarkUpdateRefsMetadataClosure cl(q, rp); + ShenandoahMarkUpdateRefsMetadataClosure cl(q, rp, old); mark_loop_work, GENERATION, CANCELLABLE>(&cl, ld, w, t); } } else { if (strdedup) { - ShenandoahMarkRefsMetadataDedupClosure cl(q, rp); + ShenandoahMarkRefsMetadataDedupClosure cl(q, rp, old); mark_loop_work, GENERATION, CANCELLABLE>(&cl, ld, w, t); } else { - ShenandoahMarkRefsMetadataClosure cl(q, rp); + ShenandoahMarkRefsMetadataClosure cl(q, rp, old); mark_loop_work, GENERATION, CANCELLABLE>(&cl, ld, w, t); } } } else { - if (heap->has_forwarded_objects()) { + if (update_refs) { if (strdedup) { - ShenandoahMarkUpdateRefsDedupClosure cl(q, rp); + ShenandoahMarkUpdateRefsDedupClosure cl(q, rp, old); mark_loop_work, GENERATION, CANCELLABLE>(&cl, ld, w, t); } else { - ShenandoahMarkUpdateRefsClosure cl(q, rp); + ShenandoahMarkUpdateRefsClosure cl(q, rp, old); mark_loop_work, GENERATION, CANCELLABLE>(&cl, ld, w, t); } } else { if (strdedup) { - ShenandoahMarkRefsDedupClosure cl(q, rp); + ShenandoahMarkRefsDedupClosure cl(q, rp, old); mark_loop_work, GENERATION, CANCELLABLE>(&cl, ld, w, t); } else { - ShenandoahMarkRefsClosure cl(q, rp); + ShenandoahMarkRefsClosure cl(q, rp, old); mark_loop_work, GENERATION, CANCELLABLE>(&cl, ld, w, t); } } @@ -143,8 +139,9 @@ void ShenandoahMark::mark_loop_work(T* cl, ShenandoahLiveData* live_data, uint w } } q = get_queue(worker_id); + ShenandoahObjToScanQueue* old = get_old_queue(worker_id); - ShenandoahSATBBufferClosure drain_satb(q); + ShenandoahSATBBufferClosure drain_satb(q, old); SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set(); /* @@ -182,20 +179,30 @@ void ShenandoahMark::mark_loop_work(T* cl, ShenandoahLiveData* live_data, uint w void ShenandoahMark::mark_loop(GenerationMode generation, uint worker_id, TaskTerminator* terminator, ShenandoahReferenceProcessor *rp, bool cancellable, bool strdedup) { + bool update_refs = ShenandoahHeap::heap()->has_forwarded_objects(); switch (generation) { case YOUNG: { if (cancellable) { - mark_loop_prework(worker_id, terminator, rp, strdedup); + mark_loop_prework(worker_id, terminator, rp, strdedup, update_refs); + } else { + mark_loop_prework(worker_id, terminator, rp, strdedup, update_refs); + } + break; + } + case OLD: { + // Old generation collection only performs marking, it should to update references. + if (cancellable) { + mark_loop_prework(worker_id, terminator, rp, strdedup, false); } else { - mark_loop_prework(worker_id, terminator, rp, strdedup); + mark_loop_prework(worker_id, terminator, rp, strdedup, false); } break; } case GLOBAL: { if (cancellable) { - mark_loop_prework(worker_id, terminator, rp, strdedup); + mark_loop_prework(worker_id, terminator, rp, strdedup, update_refs); } else { - mark_loop_prework(worker_id, terminator, rp, strdedup); + mark_loop_prework(worker_id, terminator, rp, strdedup, update_refs); } break; } @@ -204,3 +211,18 @@ void ShenandoahMark::mark_loop(GenerationMode generation, uint worker_id, TaskTe break; } } + +template<> +bool ShenandoahMark::in_generation(oop obj) { + return ShenandoahHeap::heap()->is_in_young(obj); +} + +template<> +bool ShenandoahMark::in_generation(oop obj) { + return ShenandoahHeap::heap()->is_in_old(obj); +} + +template<> +bool ShenandoahMark::in_generation(oop obj) { + return true; +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.hpp index 7d83d3c8d67a4..4ee8952052604 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMark.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.hpp @@ -29,8 +29,6 @@ #include "gc/shenandoah/shenandoahOopClosures.hpp" #include "gc/shenandoah/shenandoahTaskqueue.hpp" -class ShenandoahCMDrainMarkingStackClosure; - template class ShenandoahInitMarkRootsClosure : public OopClosure { private: @@ -51,23 +49,27 @@ class ShenandoahInitMarkRootsClosure : public OopClosure { // Mark class does not maintain states. Instead, mark states are // maintained by task queues, mark bitmap and SATB buffers (concurrent mark) class ShenandoahMark: public StackObj { - friend class ShenandoahCMDrainMarkingStackClosure; protected: + ShenandoahGeneration* const _generation; ShenandoahObjToScanQueueSet* const _task_queues; + ShenandoahObjToScanQueueSet* const _old_gen_task_queues; protected: - ShenandoahMark(); + ShenandoahMark(ShenandoahGeneration* generation); public: template - static inline void mark_through_ref(T* p, ShenandoahObjToScanQueue* q, ShenandoahMarkingContext* const mark_context, bool weak); - - static void clear(); + static inline void mark_through_ref(T* p, ShenandoahObjToScanQueue* q, ShenandoahObjToScanQueue* old, ShenandoahMarkingContext* const mark_context, bool weak); // Helpers inline ShenandoahObjToScanQueueSet* task_queues() const; + ShenandoahObjToScanQueueSet* old_task_queues() { + return _old_gen_task_queues; + } + inline ShenandoahObjToScanQueue* get_queue(uint index) const; + inline ShenandoahObjToScanQueue* get_old_queue(uint index) const; // ---------- Marking loop and tasks private: @@ -86,11 +88,21 @@ class ShenandoahMark: public StackObj { void mark_loop_work(T* cl, ShenandoahLiveData* live_data, uint worker_id, TaskTerminator *t); template - void mark_loop_prework(uint worker_id, TaskTerminator *terminator, ShenandoahReferenceProcessor *rp, bool strdedup); + void mark_loop_prework(uint worker_id, TaskTerminator *terminator, + ShenandoahReferenceProcessor *rp, bool strdedup, bool update_refs); + + template + static bool in_generation(oop obj); + + template + static void mark_ref(ShenandoahObjToScanQueue* q, + ShenandoahMarkingContext* const mark_context, bool weak, + oop obj); protected: void mark_loop(GenerationMode generation, uint worker_id, TaskTerminator* terminator, ShenandoahReferenceProcessor *rp, bool cancellable, bool strdedup); + }; #endif // SHARE_GC_SHENANDOAH_SHENANDOAHMARK_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp index 1e057fe73b248..ef5a614bf36a3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp @@ -48,13 +48,18 @@ ShenandoahInitMarkRootsClosure::ShenandoahInitMarkRootsClosure(Shena template template void ShenandoahInitMarkRootsClosure::do_oop_work(T* p) { - ShenandoahMark::mark_through_ref(p, _queue, _mark_context, false); + // Only called from STW mark, should not be used to bootstrap old generation marking. + ShenandoahMark::mark_through_ref(p, _queue, nullptr, _mark_context, false); } template void ShenandoahMark::do_task(ShenandoahObjToScanQueue* q, T* cl, ShenandoahLiveData* live_data, ShenandoahMarkTask* task) { oop obj = task->obj(); + // HEY! This will push array chunks into the mark queue with no regard for + // generations. I don't think it will break anything, but the young generation + // scan might end up processing some old generation array chunks. + shenandoah_assert_not_forwarded(NULL, obj); shenandoah_assert_marked(NULL, obj); shenandoah_assert_not_in_cset_except(NULL, obj, ShenandoahHeap::heap()->cancelled_gc()); @@ -221,18 +226,20 @@ template class ShenandoahSATBBufferClosure : public SATBBufferClosure { private: ShenandoahObjToScanQueue* _queue; + ShenandoahObjToScanQueue* _old; ShenandoahHeap* _heap; ShenandoahMarkingContext* const _mark_context; public: - ShenandoahSATBBufferClosure(ShenandoahObjToScanQueue* q) : + ShenandoahSATBBufferClosure(ShenandoahObjToScanQueue* q, ShenandoahObjToScanQueue* old) : _queue(q), + _old(old), _heap(ShenandoahHeap::heap()), _mark_context(_heap->marking_context()) { } void do_buffer(void **buffer, size_t size) { - assert(size == 0 || !_heap->has_forwarded_objects(), "Forwarded objects are not expected here"); + assert(size == 0 || !_heap->has_forwarded_objects() || _heap->is_concurrent_old_mark_in_progress(), "Forwarded objects are not expected here"); if (ShenandoahStringDedup::is_enabled()) { do_buffer_impl(buffer, size); } else { @@ -244,13 +251,13 @@ class ShenandoahSATBBufferClosure : public SATBBufferClosure { void do_buffer_impl(void **buffer, size_t size) { for (size_t i = 0; i < size; ++i) { oop *p = (oop *) &buffer[i]; - ShenandoahMark::mark_through_ref(p, _queue, _mark_context, false); + ShenandoahMark::mark_through_ref(p, _queue, _old, _mark_context, false); } } }; template -inline void ShenandoahMark::mark_through_ref(T *p, ShenandoahObjToScanQueue* q, ShenandoahMarkingContext* const mark_context, bool weak) { +inline void ShenandoahMark::mark_through_ref(T *p, ShenandoahObjToScanQueue* q, ShenandoahObjToScanQueue* old, ShenandoahMarkingContext* const mark_context, bool weak) { T o = RawAccess<>::oop_load(p); if (!CompressedOops::is_null(o)) { oop obj = CompressedOops::decode_not_null(o); @@ -258,26 +265,35 @@ inline void ShenandoahMark::mark_through_ref(T *p, ShenandoahObjToScanQueue* q, shenandoah_assert_not_forwarded(p, obj); shenandoah_assert_not_in_cset_except(p, obj, ShenandoahHeap::heap()->cancelled_gc()); - if (GENERATION != YOUNG || ShenandoahHeap::heap()->is_in_young(obj)) { - bool skip_live = false; - bool marked; - if (weak) { - marked = mark_context->mark_weak(obj); - } else { - marked = mark_context->mark_strong(obj, /* was_upgraded = */ skip_live); - } - if (marked) { - bool pushed = q->push(ShenandoahMarkTask(obj, skip_live, weak)); - assert(pushed, "overflow queue should always succeed pushing"); - - if ((STRING_DEDUP == ENQUEUE_DEDUP) && ShenandoahStringDedup::is_candidate(obj)) { - assert(ShenandoahStringDedup::is_enabled(), "Must be enabled"); - ShenandoahStringDedup::enqueue_candidate(obj); - } - } + if (in_generation(obj)) { + mark_ref(q, mark_context, weak, obj); + shenandoah_assert_marked(p, obj); + } else if (old != nullptr) { + mark_ref(old, mark_context, weak, obj); + shenandoah_assert_marked(p, obj); } + } +} - shenandoah_assert_marked(p, obj); +template +void ShenandoahMark::mark_ref(ShenandoahObjToScanQueue* q, + ShenandoahMarkingContext* const mark_context, + bool weak, oop obj) { + bool skip_live = false; + bool marked; + if (weak) { + marked = mark_context->mark_weak(obj); + } else { + marked = mark_context->mark_strong(obj, /* was_upgraded = */ skip_live); + } + if (marked) { + bool pushed = q->push(ShenandoahMarkTask(obj, skip_live, weak)); + assert(pushed, "overflow queue should always succeed pushing"); + + if ((STRING_DEDUP == ENQUEUE_DEDUP) && ShenandoahStringDedup::is_candidate(obj)) { + assert(ShenandoahStringDedup::is_enabled(), "Must be enabled"); + ShenandoahStringDedup::enqueue_candidate(obj); + } } } @@ -288,4 +304,12 @@ ShenandoahObjToScanQueueSet* ShenandoahMark::task_queues() const { ShenandoahObjToScanQueue* ShenandoahMark::get_queue(uint index) const { return _task_queues->queue(index); } + +ShenandoahObjToScanQueue* ShenandoahMark::get_old_queue(uint index) const { + if (_old_gen_task_queues != nullptr) { + return _old_gen_task_queues->queue(index); + } + return nullptr; +} + #endif // SHARE_GC_SHENANDOAH_SHENANDOAHMARK_INLINE_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.cpp new file mode 100644 index 0000000000000..40dc4549f738f --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.cpp @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2021, Red Hat, Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "gc/shenandoah/shenandoahHeap.inline.hpp" +#include "gc/shenandoah/shenandoahHeapRegion.hpp" +#include "gc/shenandoah/shenandoahMarkClosures.hpp" +#include "gc/shenandoah/shenandoahMarkingContext.hpp" +#include "gc/shenandoah/shenandoahSharedVariables.hpp" + + +ShenandoahFinalMarkUpdateRegionStateClosure::ShenandoahFinalMarkUpdateRegionStateClosure( + ShenandoahMarkingContext *ctx) : + _ctx(ctx), _lock(ShenandoahHeap::heap()->lock()) {} + +void ShenandoahFinalMarkUpdateRegionStateClosure::heap_region_do(ShenandoahHeapRegion* r) { + if (r->is_active()) { + // All allocations past TAMS are implicitly live, adjust the region data. + // Bitmaps/TAMS are swapped at this point, so we need to poll complete bitmap. + HeapWord *tams = _ctx->top_at_mark_start(r); + HeapWord *top = r->top(); + if (top > tams) { + r->increase_live_data_alloc_words(pointer_delta(top, tams)); + } + + // We are about to select the collection set, make sure it knows about + // current pinning status. Also, this allows trashing more regions that + // now have their pinning status dropped. + if (r->is_pinned()) { + if (r->pin_count() == 0) { + ShenandoahHeapLocker locker(_lock); + r->make_unpinned(); + } + } else { + if (r->pin_count() > 0) { + ShenandoahHeapLocker locker(_lock); + r->make_pinned(); + } + } + + if (ShenandoahHeap::heap()->mode()->is_generational()) { + // HEY! Allocations move the watermark when top moves, however compacting + // objects will sometimes lower top beneath the watermark, after which, + // attempts to read the watermark will assert out (watermark should not be + // higher than top). I think the right way™ to check for new allocations + // is to compare top with the TAMS as is done earlier in this function. + // if (r->top() != r->get_update_watermark()) { + if (top > tams) { + // There have been allocations in this region since the start of the cycle. + // Any objects new to this region must not assimilate elevated age. + r->reset_age(); + } else { + r->increment_age(); + } + } + + // Remember limit for updating refs. It's guaranteed that we get no + // from-space-refs written from here on. + r->set_update_watermark_at_safepoint(r->top()); + } else { + assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index()); + assert(_ctx->top_at_mark_start(r) == r->top(), + "Region " SIZE_FORMAT " should have correct TAMS", r->index()); + } +} + diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.hpp new file mode 100644 index 0000000000000..31817c61f0d67 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.hpp @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2021, Red Hat, Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHMARKCLOSURES_HPP +#define SHARE_GC_SHENANDOAH_SHENANDOAHMARKCLOSURES_HPP + +#include "gc/shenandoah/shenandoahHeap.hpp" + +class ShenandoahMarkingContext; +class ShenandoahHeapRegion; + +class ShenandoahFinalMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure { +private: + ShenandoahMarkingContext* const _ctx; + ShenandoahHeapLock* const _lock; +public: + ShenandoahFinalMarkUpdateRegionStateClosure(ShenandoahMarkingContext* ctx); + + void heap_region_do(ShenandoahHeapRegion* r); + + bool is_thread_safe() { return true; } +}; + +#endif // SHARE_GC_SHENANDOAH_SHENANDOAHMARKCLOSURES_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.cpp index 5eef4c4dadc86..85350e9a3eece 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.cpp @@ -25,32 +25,14 @@ #include "precompiled.hpp" #include "gc/shared/markBitMap.inline.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" -#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" #include "gc/shenandoah/shenandoahMarkingContext.hpp" -#include "gc/shenandoah/shenandoahTaskqueue.inline.hpp" -#include "utilities/stack.inline.hpp" -ShenandoahMarkingContext::ShenandoahMarkingContext(MemRegion heap_region, MemRegion bitmap_region, size_t num_regions, uint max_queues) : +ShenandoahMarkingContext::ShenandoahMarkingContext(MemRegion heap_region, MemRegion bitmap_region, size_t num_regions) : _mark_bit_map(heap_region, bitmap_region), _top_bitmaps(NEW_C_HEAP_ARRAY(HeapWord*, num_regions, mtGC)), _top_at_mark_starts_base(NEW_C_HEAP_ARRAY(HeapWord*, num_regions, mtGC)), _top_at_mark_starts(_top_at_mark_starts_base - - ((uintx) heap_region.start() >> ShenandoahHeapRegion::region_size_bytes_shift())), - _task_queues(new ShenandoahObjToScanQueueSet(max_queues)) { - assert(max_queues > 0, "At least one queue"); - for (uint i = 0; i < max_queues; ++i) { - ShenandoahObjToScanQueue* task_queue = new ShenandoahObjToScanQueue(); - task_queue->initialize(); - _task_queues->register_queue(i, task_queue); - } -} - -ShenandoahMarkingContext::~ShenandoahMarkingContext() { - for (uint i = 0; i < _task_queues->size(); ++i) { - ShenandoahObjToScanQueue* q = _task_queues->queue(i); - delete q; - } - delete _task_queues; + ((uintx) heap_region.start() >> ShenandoahHeapRegion::region_size_bytes_shift())) { } bool ShenandoahMarkingContext::is_bitmap_clear() const { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.hpp index d58117c02e2d6..b39be1fb2d659 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.hpp @@ -47,12 +47,8 @@ class ShenandoahMarkingContext : public CHeapObj { ShenandoahSharedFlag _is_complete; - // Marking task queues - ShenandoahObjToScanQueueSet* _task_queues; - public: - ShenandoahMarkingContext(MemRegion heap_region, MemRegion bitmap_region, size_t num_regions, uint max_queues); - ~ShenandoahMarkingContext(); + ShenandoahMarkingContext(MemRegion heap_region, MemRegion bitmap_region, size_t num_regions); /* * Marks the object. Returns true if the object has not been marked before and has @@ -66,6 +62,8 @@ class ShenandoahMarkingContext : public CHeapObj { inline bool is_marked(oop) const; inline bool is_marked_strong(oop obj) const; inline bool is_marked_weak(oop obj) const; + inline bool is_marked_or_old(oop obj) const; + inline bool is_marked_strong_or_old(oop obj) const; inline HeapWord* get_next_marked_addr(HeapWord* addr, HeapWord* limit) const; @@ -86,9 +84,6 @@ class ShenandoahMarkingContext : public CHeapObj { bool is_complete(); void mark_complete(); void mark_incomplete(); - - // Task queues - ShenandoahObjToScanQueueSet* task_queues() const { return _task_queues; } }; #endif // SHARE_GC_SHENANDOAH_SHENANDOAHMARKINGCONTEXT_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.inline.hpp index f829d2aa232d4..0bf9e500d0f50 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.inline.hpp @@ -48,6 +48,14 @@ inline bool ShenandoahMarkingContext::is_marked_weak(oop obj) const { return allocated_after_mark_start(obj) || _mark_bit_map.is_marked_weak(cast_from_oop(obj)); } +inline bool ShenandoahMarkingContext::is_marked_or_old(oop obj) const { + return is_marked(obj) || ShenandoahHeap::heap()->is_old(obj); +} + +inline bool ShenandoahMarkingContext::is_marked_strong_or_old(oop obj) const { + return is_marked_strong(obj) || ShenandoahHeap::heap()->is_old(obj); +} + inline HeapWord* ShenandoahMarkingContext::get_next_marked_addr(HeapWord* start, HeapWord* limit) const { return _mark_bit_map.get_next_marked_addr(start, limit); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp b/src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp index a35d54259dd4c..d3fdc7caabc16 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp @@ -155,13 +155,13 @@ void ShenandoahNMethod::heal_nmethod(nmethod* nm) { assert(data->lock()->owned_by_self(), "Must hold the lock"); ShenandoahHeap* const heap = ShenandoahHeap::heap(); - if (heap->is_concurrent_mark_in_progress()) { - ShenandoahKeepAliveClosure cl; - data->oops_do(&cl); - } else if (heap->is_concurrent_weak_root_in_progress() || - heap->is_concurrent_strong_root_in_progress()) { + if (heap->is_concurrent_weak_root_in_progress() || + heap->is_concurrent_strong_root_in_progress()) { ShenandoahEvacOOMScope evac_scope; heal_nmethod_metadata(data); + } else if (heap->is_concurrent_mark_in_progress()) { + ShenandoahKeepAliveClosure cl; + data->oops_do(&cl); } else { // There is possibility that GC is cancelled when it arrives final mark. // In this case, concurrent root phase is skipped and degenerated GC should be diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp new file mode 100644 index 0000000000000..f07543d923e79 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2021, Amazon.com, Inc. or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "gc/shenandoah/shenandoahFreeSet.hpp" +#include "gc/shenandoah/shenandoahHeap.inline.hpp" +#include "gc/shenandoah/shenandoahOldGC.hpp" +#include "gc/shenandoah/shenandoahOopClosures.inline.hpp" + +ShenandoahOldGC::ShenandoahOldGC(ShenandoahGeneration* generation, ShenandoahSharedFlag& allow_preemption) : + ShenandoahConcurrentGC(generation), _allow_preemption(allow_preemption) {} + +bool ShenandoahOldGC::collect(GCCause::Cause cause) { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + + // Continue concurrent mark, do not reset regions, do not mark roots, do not collect $200. + _allow_preemption.set(); + entry_mark(); + _allow_preemption.unset(); + if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark)) return false; + + // Complete marking under STW + vmop_entry_final_mark(); + + // We aren't dealing with old generation evacuation yet. Our heuristic + // should not have built a cset in final mark. + assert(!heap->is_evacuation_in_progress(), "Old gen evacuations are not supported"); + + // Concurrent stack processing + if (heap->is_evacuation_in_progress()) { + entry_thread_roots(); + } + + // Process weak roots that might still point to regions that would be broken by cleanup + if (heap->is_concurrent_weak_root_in_progress()) { + entry_weak_refs(); + entry_weak_roots(); + } + + // Final mark might have reclaimed some immediate garbage, kick cleanup to reclaim + // the space. This would be the last action if there is nothing to evacuate. + entry_cleanup_early(); + + { + ShenandoahHeapLocker locker(heap->lock()); + heap->free_set()->log_status(); + } + + // Perform concurrent class unloading + if (heap->unload_classes() && + heap->is_concurrent_weak_root_in_progress()) { + entry_class_unloading(); + } + + // Processing strong roots + // This may be skipped if there is nothing to update/evacuate. + // If so, strong_root_in_progress would be unset. + if (heap->is_concurrent_strong_root_in_progress()) { + entry_strong_roots(); + } + + entry_rendezvous_roots(); + return true; +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.hpp new file mode 100644 index 0000000000000..b6598cda05906 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.hpp @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2021, Amazon.com, Inc. or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHOLDGC_HPP +#define SHARE_GC_SHENANDOAH_SHENANDOAHOLDGC_HPP + +#include "gc/shared/gcCause.hpp" +#include "gc/shenandoah/shenandoahConcurrentGC.hpp" + +class ShenandoahGeneration; + +class ShenandoahOldGC : public ShenandoahConcurrentGC { + public: + ShenandoahOldGC(ShenandoahGeneration* generation, ShenandoahSharedFlag& allow_preemption); + bool collect(GCCause::Cause cause); + private: + ShenandoahSharedFlag& _allow_preemption; +}; + + +#endif //SHARE_GC_SHENANDOAH_SHENANDOAHOLDGC_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp new file mode 100644 index 0000000000000..f110953f2675a --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2021, Amazon.com, Inc. or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "gc/shared/strongRootsScope.hpp" +#include "gc/shenandoah/shenandoahAsserts.hpp" +#include "gc/shenandoah/shenandoahHeap.hpp" +#include "gc/shenandoah/shenandoahHeap.inline.hpp" +#include "gc/shenandoah/shenandoahHeapRegion.hpp" +#include "gc/shenandoah/shenandoahMark.inline.hpp" +#include "gc/shenandoah/shenandoahOldGeneration.hpp" +#include "gc/shenandoah/shenandoahOopClosures.inline.hpp" +#include "gc/shenandoah/shenandoahStringDedup.hpp" +#include "gc/shenandoah/shenandoahUtils.hpp" + +class ShenandoahFlushAllSATB : public ThreadClosure { + private: + SATBMarkQueueSet& _satb_qset; + uintx _claim_token; + + public: + explicit ShenandoahFlushAllSATB(SATBMarkQueueSet& satb_qset) : + _satb_qset(satb_qset), + _claim_token(Threads::thread_claim_token()) {} + + void do_thread(Thread* thread) { + if (thread->claim_threads_do(true, _claim_token)) { + // Transfer any partial buffer to the qset for completed buffer processing. + _satb_qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread)); + } + } +}; + +class ShenandoahProcessOldSATB : public SATBBufferClosure { + private: + ShenandoahObjToScanQueue* _queue; + ShenandoahHeap* _heap; + ShenandoahMarkingContext* const _mark_context; + + public: + size_t _trashed_oops; + + explicit ShenandoahProcessOldSATB(ShenandoahObjToScanQueue* q) : + _queue(q), + _heap(ShenandoahHeap::heap()), + _mark_context(_heap->marking_context()), + _trashed_oops(0) {} + + void do_buffer(void **buffer, size_t size) { + assert(size == 0 || !_heap->has_forwarded_objects() || _heap->is_concurrent_old_mark_in_progress(), "Forwarded objects are not expected here"); + if (ShenandoahStringDedup::is_enabled()) { + do_buffer_impl(buffer, size); + } else { + do_buffer_impl(buffer, size); + } + } + + template + void do_buffer_impl(void **buffer, size_t size) { + for (size_t i = 0; i < size; ++i) { + oop *p = (oop *) &buffer[i]; + ShenandoahHeapRegion* region = _heap->heap_region_containing(*p); + if (!region->is_trash()) { + ShenandoahMark::mark_through_ref(p, _queue, NULL, _mark_context, false); + } else { + ++_trashed_oops; + } + } + } +}; + +class ShenandoahPurgeSATBTask : public AbstractGangTask { + private: + ShenandoahObjToScanQueueSet* _mark_queues; + public: + size_t _trashed_oops; + + explicit ShenandoahPurgeSATBTask(ShenandoahObjToScanQueueSet* queues) : + AbstractGangTask("Purge SATB"), _mark_queues(queues), _trashed_oops(0) {} + + ~ShenandoahPurgeSATBTask() { + if (_trashed_oops > 0) { + log_info(gc)("Purged " SIZE_FORMAT " oops from old generation SATB buffers.", _trashed_oops); + } + } + + void work(uint worker_id) { + ShenandoahParallelWorkerSession worker_session(worker_id); + ShenandoahSATBMarkQueueSet &satb_queues = ShenandoahBarrierSet::satb_mark_queue_set(); + ShenandoahFlushAllSATB flusher(satb_queues); + Threads::threads_do(&flusher); + + ShenandoahObjToScanQueue* mark_queue = _mark_queues->queue(worker_id); + ShenandoahProcessOldSATB processor(mark_queue); + while (satb_queues.apply_closure_to_completed_buffer(&processor)) {} + + Atomic::add(&_trashed_oops, processor._trashed_oops); + } +}; + +ShenandoahOldGeneration::ShenandoahOldGeneration(uint max_queues, size_t max_capacity, size_t soft_max_capacity) + : ShenandoahGeneration(OLD, max_queues, max_capacity, soft_max_capacity) {} + +const char* ShenandoahOldGeneration::name() const { + return "OLD"; +} + +bool ShenandoahOldGeneration::contains(ShenandoahHeapRegion* region) const { + return region->affiliation() != YOUNG_GENERATION; +} + +void ShenandoahOldGeneration::parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl) { + ShenandoahGenerationRegionClosure old_regions(cl); + ShenandoahHeap::heap()->parallel_heap_region_iterate(&old_regions); +} + +void ShenandoahOldGeneration::set_concurrent_mark_in_progress(bool in_progress) { + ShenandoahHeap::heap()->set_concurrent_old_mark_in_progress(in_progress); +} + +bool ShenandoahOldGeneration::is_concurrent_mark_in_progress() { + return ShenandoahHeap::heap()->is_concurrent_old_mark_in_progress(); +} + +void ShenandoahOldGeneration::purge_satb_buffers(bool abandon) { + ShenandoahHeap *heap = ShenandoahHeap::heap(); + shenandoah_assert_safepoint(); + assert(heap->is_concurrent_old_mark_in_progress(), "Only necessary during old marking."); + + if (abandon) { + ShenandoahBarrierSet::satb_mark_queue_set().abandon_partial_marking(); + } else { + uint nworkers = heap->workers()->active_workers(); + StrongRootsScope scope(nworkers); + + ShenandoahPurgeSATBTask purge_satb_task(task_queues()); + heap->workers()->run_task(&purge_satb_task); + } +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp new file mode 100644 index 0000000000000..70052c7c7bd6d --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp @@ -0,0 +1,67 @@ +/* + * Copyright (c) 2021, Amazon.com, Inc. or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_SHENANDOAH_SHENANDOAHOLDGENERATION_HPP +#define SHARE_VM_GC_SHENANDOAH_SHENANDOAHOLDGENERATION_HPP + +#include "gc/shenandoah/shenandoahGeneration.hpp" + +class ShenandoahHeapRegion; +class ShenandoahHeapRegionClosure; + +class ShenandoahOldGeneration : public ShenandoahGeneration { + public: + ShenandoahOldGeneration(uint max_queues, size_t max_capacity, size_t soft_max_capacity); + + const char* name() const; + + bool contains(ShenandoahHeapRegion* region) const; + void parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl); + void set_concurrent_mark_in_progress(bool in_progress); + + // We leave the SATB barrier on for the entirety of the old generation + // marking phase. In some cases, this can cause a write to a perfectly + // reachable oop to enqueue a pointer that later becomes garbage (because + // it points at an object in the collection set, for example). There are + // also cases where the referent of a weak reference ends up in the SATB + // and is later collected. In these cases the oop in the SATB buffer becomes + // invalid and the _next_ cycle will crash during its marking phase. To + // avoid this problem, we "purge" the SATB buffers during the final update + // references phase if (and only if) an old generation mark is in progress. + // At this stage we can safely determine if any of the oops in the SATB + // buffer belong to trashed regions (before they are recycled). As it + // happens, flushing a SATB queue also filters out oops which have already + // been marked - which is the case for anything that is being evacuated + // from the collection set. + // + // Alternatively, we could inspect the state of the heap and the age of the + // object at the barrier, but we reject this approach because it is likely + // the performance impact would be too severe. + void purge_satb_buffers(bool abandon); + protected: + bool is_concurrent_mark_in_progress(); +}; + + +#endif //SHARE_VM_GC_SHENANDOAH_SHENANDOAHOLDGENERATION_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp index 1467bd85816dd..63dfc28caccfc 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp @@ -33,11 +33,6 @@ #include "memory/iterator.hpp" #include "runtime/thread.hpp" -enum GenerationMode { - YOUNG, - GLOBAL -}; - enum StringDedupMode { NO_DEDUP, // Do not do anything for String deduplication ENQUEUE_DEDUP // Enqueue candidate Strings for deduplication @@ -46,6 +41,7 @@ enum StringDedupMode { class ShenandoahMarkRefsSuperClosure : public MetadataVisitingOopIterateClosure { private: ShenandoahObjToScanQueue* _queue; + ShenandoahObjToScanQueue* _old_queue; ShenandoahMarkingContext* const _mark_context; bool _weak; @@ -54,7 +50,7 @@ class ShenandoahMarkRefsSuperClosure : public MetadataVisitingOopIterateClosure void work(T *p); public: - ShenandoahMarkRefsSuperClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp); + ShenandoahMarkRefsSuperClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp, ShenandoahObjToScanQueue* old_queue = NULL); bool is_weak() const { return _weak; @@ -73,8 +69,8 @@ class ShenandoahMarkUpdateRefsSuperClosure : public ShenandoahMarkRefsSuperClosu inline void work(T* p); public: - ShenandoahMarkUpdateRefsSuperClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) : - ShenandoahMarkRefsSuperClosure(q, rp), + ShenandoahMarkUpdateRefsSuperClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp, ShenandoahObjToScanQueue* old = NULL) : + ShenandoahMarkRefsSuperClosure(q, rp, old), _heap(ShenandoahHeap::heap()) { assert(_heap->is_stw_gc_in_progress(), "Can only be used for STW GC"); }; @@ -87,8 +83,8 @@ class ShenandoahMarkUpdateRefsClosure : public ShenandoahMarkUpdateRefsSuperClos inline void do_oop_work(T* p) { work(p); } public: - ShenandoahMarkUpdateRefsClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) : - ShenandoahMarkUpdateRefsSuperClosure(q, rp) {} + ShenandoahMarkUpdateRefsClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp, ShenandoahObjToScanQueue* old = NULL) : + ShenandoahMarkUpdateRefsSuperClosure(q, rp, old) {} virtual void do_oop(narrowOop* p) { do_oop_work(p); } virtual void do_oop(oop* p) { do_oop_work(p); } @@ -102,8 +98,8 @@ class ShenandoahMarkUpdateRefsDedupClosure : public ShenandoahMarkUpdateRefsSupe inline void do_oop_work(T* p) { work(p); } public: - ShenandoahMarkUpdateRefsDedupClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) : - ShenandoahMarkUpdateRefsSuperClosure(q, rp) {} + ShenandoahMarkUpdateRefsDedupClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp, ShenandoahObjToScanQueue* old = NULL) : + ShenandoahMarkUpdateRefsSuperClosure(q, rp, old) {} virtual void do_oop(narrowOop* p) { do_oop_work(p); } virtual void do_oop(oop* p) { do_oop_work(p); } @@ -117,8 +113,8 @@ class ShenandoahMarkUpdateRefsMetadataClosure : public ShenandoahMarkUpdateRefsS inline void do_oop_work(T* p) { work(p); } public: - ShenandoahMarkUpdateRefsMetadataClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) : - ShenandoahMarkUpdateRefsSuperClosure(q, rp) {} + ShenandoahMarkUpdateRefsMetadataClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp, ShenandoahObjToScanQueue* old = NULL) : + ShenandoahMarkUpdateRefsSuperClosure(q, rp, old) {} virtual void do_oop(narrowOop* p) { do_oop_work(p); } virtual void do_oop(oop* p) { do_oop_work(p); } @@ -132,8 +128,8 @@ class ShenandoahMarkUpdateRefsMetadataDedupClosure : public ShenandoahMarkUpdate inline void do_oop_work(T* p) { work(p); } public: - ShenandoahMarkUpdateRefsMetadataDedupClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) : - ShenandoahMarkUpdateRefsSuperClosure(q, rp) {} + ShenandoahMarkUpdateRefsMetadataDedupClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp, ShenandoahObjToScanQueue* old = NULL) : + ShenandoahMarkUpdateRefsSuperClosure(q, rp, old) {} virtual void do_oop(narrowOop* p) { do_oop_work(p); } virtual void do_oop(oop* p) { do_oop_work(p); } @@ -147,8 +143,8 @@ class ShenandoahMarkRefsClosure : public ShenandoahMarkRefsSuperClosure { inline void do_oop_work(T* p) { work(p); } public: - ShenandoahMarkRefsClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) : - ShenandoahMarkRefsSuperClosure(q, rp) {}; + ShenandoahMarkRefsClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp, ShenandoahObjToScanQueue* old = NULL) : + ShenandoahMarkRefsSuperClosure(q, rp, old) {}; virtual void do_oop(narrowOop* p) { do_oop_work(p); } virtual void do_oop(oop* p) { do_oop_work(p); } @@ -162,8 +158,8 @@ class ShenandoahMarkRefsDedupClosure : public ShenandoahMarkRefsSuperClosure { inline void do_oop_work(T* p) { work(p); } public: - ShenandoahMarkRefsDedupClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) : - ShenandoahMarkRefsSuperClosure(q, rp) {}; + ShenandoahMarkRefsDedupClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp, ShenandoahObjToScanQueue* old = NULL) : + ShenandoahMarkRefsSuperClosure(q, rp, old) {}; virtual void do_oop(narrowOop* p) { do_oop_work(p); } virtual void do_oop(oop* p) { do_oop_work(p); } @@ -177,8 +173,8 @@ class ShenandoahMarkRefsMetadataClosure : public ShenandoahMarkRefsSuperClosure inline void do_oop_work(T* p) { work(p); } public: - ShenandoahMarkRefsMetadataClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) : - ShenandoahMarkRefsSuperClosure(q, rp) {}; + ShenandoahMarkRefsMetadataClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp, ShenandoahObjToScanQueue* old = NULL) : + ShenandoahMarkRefsSuperClosure(q, rp, old) {}; virtual void do_oop(narrowOop* p) { do_oop_work(p); } virtual void do_oop(oop* p) { do_oop_work(p); } @@ -192,8 +188,8 @@ class ShenandoahMarkRefsMetadataDedupClosure : public ShenandoahMarkRefsSuperClo inline void do_oop_work(T* p) { work(p); } public: - ShenandoahMarkRefsMetadataDedupClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp) : - ShenandoahMarkRefsSuperClosure(q, rp) {}; + ShenandoahMarkRefsMetadataDedupClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp, ShenandoahObjToScanQueue* old = NULL) : + ShenandoahMarkRefsSuperClosure(q, rp, old) {}; virtual void do_oop(narrowOop* p) { do_oop_work(p); } virtual void do_oop(oop* p) { do_oop_work(p); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp index 7ecda1ffc391b..560cb0932f596 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp @@ -30,7 +30,7 @@ template inline void ShenandoahMarkRefsSuperClosure::work(T* p) { - ShenandoahMark::mark_through_ref(p, _queue, _mark_context, _weak); + ShenandoahMark::mark_through_ref(p, _queue, _old_queue, _mark_context, _weak); } template diff --git a/src/hotspot/share/gc/shenandoah/shenandoahPadding.hpp b/src/hotspot/share/gc/shenandoah/shenandoahPadding.hpp index b6038d980c39b..6c11c402a2845 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahPadding.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahPadding.hpp @@ -25,6 +25,8 @@ #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHPADDING_INLINE_HPP #define SHARE_GC_SHENANDOAH_SHENANDOAHPADDING_INLINE_HPP +#include "memory/padded.hpp" + // 64 bytes is enough to cover all existing architectures. If we have some // other platforms, we would need to provide the architecture-specific // versions here. Shared code provides DEFAULT_CACHE_LINE_SIZE, which is diff --git a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp index 8d411891dcf77..b490de9d05017 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp @@ -99,8 +99,8 @@ class outputStream; f(conc_rendezvous_roots, "Rendezvous") \ f(conc_evac, "Concurrent Evacuation") \ \ - f(init_update_refs_gross, "Pause Init Update Refs (G)") \ - f(init_update_refs, "Pause Init Update Refs (N)") \ + f(init_update_refs_gross, "Pause Init Update Refs (G)") \ + f(init_update_refs, "Pause Init Update Refs (N)") \ f(init_update_refs_manage_gclabs, " Manage GCLABs") \ \ f(conc_update_refs, "Concurrent Update Refs") \ @@ -112,6 +112,7 @@ class outputStream; f(final_update_refs_update_region_states, " Update Region States") \ f(final_update_refs_trash_cset, " Trash Collection Set") \ f(final_update_refs_rebuild_freeset, " Rebuild Free Set") \ + f(final_update_refs_promote_tenured_regions, " Promote Tenured Regions") \ \ f(conc_cleanup_complete, "Concurrent Cleanup") \ \ diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.cpp new file mode 100644 index 0000000000000..5f8af48bc0f30 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.cpp @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2021, Amazon.com, Inc. and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ +#include "precompiled.hpp" + +#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" +#include "gc/shenandoah/mode/shenandoahMode.hpp" +#include "gc/shenandoah/shenandoahControlThread.hpp" +#include "gc/shenandoah/shenandoahGeneration.hpp" +#include "gc/shenandoah/shenandoahHeap.inline.hpp" +#include "gc/shenandoah/shenandoahRegulatorThread.hpp" +#include "gc/shenandoah/shenandoahYoungGeneration.hpp" +#include "logging/log.hpp" + +static ShenandoahHeuristics* get_heuristics(ShenandoahGeneration* nullable) { + return nullable != NULL ? nullable->heuristics() : NULL; +} + +ShenandoahRegulatorThread::ShenandoahRegulatorThread(ShenandoahControlThread* control_thread) : + ConcurrentGCThread(), + _control_thread(control_thread), + _sleep(ShenandoahControlIntervalMin), + _last_sleep_adjust_time(os::elapsedTime()) { + + ShenandoahHeap* heap = ShenandoahHeap::heap(); + _old_heuristics = get_heuristics(heap->old_generation()); + _young_heuristics = get_heuristics(heap->young_generation()); + _global_heuristics = get_heuristics(heap->global_generation()); + + create_and_start(); +} + +void ShenandoahRegulatorThread::run_service() { + if (ShenandoahHeap::heap()->mode()->is_generational()) { + if (ShenandoahAllowOldMarkingPreemption) { + regulate_concurrent_cycles(); + } else { + regulate_interleaved_cycles(); + } + } else { + regulate_heap(); + } + + log_info(gc)("%s: Done.", name()); +} + +void ShenandoahRegulatorThread::regulate_concurrent_cycles() { + assert(_young_heuristics != NULL, "Need young heuristics."); + assert(_old_heuristics != NULL, "Need old heuristics."); + + while (!should_terminate()) { + ShenandoahControlThread::GCMode mode = _control_thread->gc_mode(); + if (mode == ShenandoahControlThread::none) { + if (start_old_cycle()) { + log_info(gc)("Heuristics request for old collection accepted"); + } else if (start_young_cycle()) { + log_info(gc)("Heuristics request for young collection accepted"); + } + } else if (mode == ShenandoahControlThread::marking_old) { + if (start_young_cycle()) { + log_info(gc)("Heuristics request for young collection accepted"); + } + } + + regulator_sleep(); + } +} + +void ShenandoahRegulatorThread::regulate_interleaved_cycles() { + assert(_young_heuristics != NULL, "Need young heuristics."); + assert(_global_heuristics != NULL, "Need global heuristics."); + + while (!should_terminate()) { + if (start_global_cycle()) { + log_info(gc)("Heuristics request for global collection accepted."); + } else if (start_young_cycle()) { + log_info(gc)("Heuristics request for young collection accepted."); + } + + regulator_sleep(); + } +} + +void ShenandoahRegulatorThread::regulate_heap() { + assert(_global_heuristics != NULL, "Need global heuristics."); + + while (!should_terminate()) { + if (start_global_cycle()) { + log_info(gc)("Heuristics request for global collection accepted."); + } + + regulator_sleep(); + } +} + +void ShenandoahRegulatorThread::regulator_sleep() { + // Wait before performing the next action. If allocation happened during this wait, + // we exit sooner, to let heuristics re-evaluate new conditions. If we are at idle, + // back off exponentially. + double current = os::elapsedTime(); + + if (_heap_changed.try_unset()) { + _sleep = ShenandoahControlIntervalMin; + } else if ((current - _last_sleep_adjust_time) * 1000 > ShenandoahControlIntervalAdjustPeriod){ + _sleep = MIN2(ShenandoahControlIntervalMax, MAX2(1, _sleep * 2)); + _last_sleep_adjust_time = current; + } + + os::naked_short_sleep(_sleep); +} + +bool ShenandoahRegulatorThread::start_old_cycle() { + return _old_heuristics->should_start_gc() && _control_thread->request_concurrent_gc(OLD); +} + +bool ShenandoahRegulatorThread::start_young_cycle() { + return _young_heuristics->should_start_gc() && _control_thread->request_concurrent_gc(YOUNG); +} + +bool ShenandoahRegulatorThread::start_global_cycle() { + return _global_heuristics->should_start_gc() && _control_thread->request_concurrent_gc(GLOBAL); +} + +void ShenandoahRegulatorThread::stop_service() { + log_info(gc)("%s: Stop requested.", name()); +} + diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.hpp b/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.hpp new file mode 100644 index 0000000000000..af964ddf90989 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.hpp @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2021, Amazon.com, Inc. and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ +#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHREGULATORTHREAD_HPP +#define SHARE_GC_SHENANDOAH_SHENANDOAHREGULATORTHREAD_HPP + +#include "gc/shared/concurrentGCThread.hpp" +#include "gc/shared/gcCause.hpp" +#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" +#include "gc/shenandoah/shenandoahHeap.hpp" +#include "runtime/mutex.hpp" + +/* + * The purpose of this class (and thread) is to allow us to continue + * to evaluate heuristics during a garbage collection. This is necessary + * to allow young generation collections to interrupt and old generation + * collection which is in-progress. This puts heuristic triggers on the + * same footing as other gc requests (alloc failure, System.gc, etc.). + * However, this regulator does not block after submitting a gc request. + * + * We could use a PeriodicTask for this, but this thread will sleep longer + * when the allocation rate is lower and PeriodicTasks cannot adjust their + * sleep time. + */ +class ShenandoahRegulatorThread: public ConcurrentGCThread { + friend class VMStructs; + + public: + explicit ShenandoahRegulatorThread(ShenandoahControlThread* control_thread); + + char* name() const { return (char*)"ShenandoahRegulatorThread";} + + // This is called from allocation path, and thus should be fast. + void notify_heap_changed() { + // Notify that something had changed. + if (_heap_changed.is_unset()) { + _heap_changed.set(); + } + } + + protected: + void run_service(); + void stop_service(); + + private: + void regulate_interleaved_cycles(); + void regulate_concurrent_cycles(); + void regulate_heap(); + + bool start_old_cycle(); + bool start_young_cycle(); + bool start_global_cycle(); + + ShenandoahSharedFlag _heap_changed; + ShenandoahControlThread* _control_thread; + ShenandoahHeuristics* _young_heuristics; + ShenandoahHeuristics* _old_heuristics; + ShenandoahHeuristics* _global_heuristics; + + int _sleep; + double _last_sleep_adjust_time; + + void regulator_sleep(); +}; + + +#endif // SHARE_GC_SHENANDOAH_SHENANDOAHREGULATORTHREAD_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp index a68bc95973b53..b8341dd5e29f1 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp @@ -29,6 +29,7 @@ #include "gc/shared/taskTerminator.hpp" #include "gc/shared/workgroup.hpp" #include "gc/shenandoah/shenandoahClosures.inline.hpp" +#include "gc/shenandoah/shenandoahGeneration.hpp" #include "gc/shenandoah/shenandoahMark.inline.hpp" #include "gc/shenandoah/shenandoahOopClosures.inline.hpp" #include "gc/shenandoah/shenandoahReferenceProcessor.hpp" @@ -56,10 +57,10 @@ void ShenandoahSTWMarkTask::work(uint worker_id) { _mark->finish_mark(worker_id); } -ShenandoahSTWMark::ShenandoahSTWMark(bool full_gc) : - ShenandoahMark(), +ShenandoahSTWMark::ShenandoahSTWMark(ShenandoahGeneration* generation, bool full_gc) : + ShenandoahMark(generation), _root_scanner(full_gc ? ShenandoahPhaseTimings::full_gc_mark : ShenandoahPhaseTimings::degen_gc_stw_mark), - _terminator(ShenandoahHeap::heap()->workers()->active_workers(), ShenandoahHeap::heap()->marking_context()->task_queues()), + _terminator(ShenandoahHeap::heap()->workers()->active_workers(), task_queues()), _full_gc(full_gc) { assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a Shenandoah safepoint"); } @@ -91,7 +92,7 @@ void ShenandoahSTWMark::mark() { assert(task_queues()->is_empty(), "Should be empty"); } - heap->mark_complete_marking_context(); + heap->global_generation()->set_mark_complete(); assert(task_queues()->is_empty(), "Should be empty"); TASKQUEUE_STATS_ONLY(task_queues()->print_taskqueue_stats()); @@ -99,8 +100,21 @@ void ShenandoahSTWMark::mark() { } void ShenandoahSTWMark::mark_roots(uint worker_id) { - ShenandoahInitMarkRootsClosure init_mark(task_queues()->queue(worker_id)); - _root_scanner.roots_do(&init_mark, worker_id); + switch (_generation->generation_mode()) { + case GLOBAL: { + ShenandoahInitMarkRootsClosure init_mark(task_queues()->queue(worker_id)); + _root_scanner.roots_do(&init_mark, worker_id); + break; + } + case YOUNG: { + ShenandoahInitMarkRootsClosure init_mark(task_queues()->queue(worker_id)); + _root_scanner.roots_do(&init_mark, worker_id); + _generation->scan_remembered_set(); + break; + } + default: + ShouldNotReachHere(); + } } void ShenandoahSTWMark::finish_mark(uint worker_id) { @@ -108,7 +122,8 @@ void ShenandoahSTWMark::finish_mark(uint worker_id) { ShenandoahWorkerTimingsTracker timer(phase, ShenandoahPhaseTimings::ParallelMark, worker_id); ShenandoahReferenceProcessor* rp = ShenandoahHeap::heap()->ref_processor(); - mark_loop(GLOBAL, worker_id, &_terminator, rp, + mark_loop(_generation->generation_mode(), + worker_id, &_terminator, rp, false, // not cancellable ShenandoahStringDedup::is_enabled()); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.hpp b/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.hpp index 771e36e0ec1fc..265f404f7b8b2 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.hpp @@ -28,6 +28,7 @@ #include "gc/shenandoah/shenandoahMark.hpp" class ShenandoahSTWMarkTask; +class ShenandoahGeneration; class ShenandoahSTWMark : public ShenandoahMark { friend class ShenandoahSTWMarkTask; @@ -37,7 +38,7 @@ class ShenandoahSTWMark : public ShenandoahMark { TaskTerminator _terminator; bool _full_gc; public: - ShenandoahSTWMark(bool full_gc); + ShenandoahSTWMark(ShenandoahGeneration* generation, bool full_gc); void mark(); private: diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp index bc480672074f3..c0891aafe6419 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp @@ -26,14 +26,16 @@ #include "precompiled.hpp" -#include "gc/shenandoah/shenandoahScanRemembered.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" +#include "gc/shenandoah/shenandoahOopClosures.inline.hpp" +#include "gc/shenandoah/shenandoahReferenceProcessor.hpp" +#include "gc/shenandoah/shenandoahScanRemembered.inline.hpp" ShenandoahDirectCardMarkRememberedSet::ShenandoahDirectCardMarkRememberedSet(CardTable *card_table, size_t total_card_count) { _heap = ShenandoahHeap::heap(); _card_table = card_table; _total_card_count = total_card_count; - _cluster_count = uint32_t(total_card_count / ShenandoahCardCluster::CardsPerCluster); + _cluster_count = total_card_count / ShenandoahCardCluster::CardsPerCluster; _card_shift = CardTable::card_shift; _byte_map = _card_table->byte_for_index(0); @@ -56,23 +58,23 @@ ShenandoahDirectCardMarkRememberedSet::~ShenandoahDirectCardMarkRememberedSet() free(_overreach_map); } -void ShenandoahDirectCardMarkRememberedSet::initialize_overreach(uint32_t first_cluster, uint32_t count) { +void ShenandoahDirectCardMarkRememberedSet::initialize_overreach(size_t first_cluster, size_t count) { // We can make this run faster in the future by explicitly // unrolling the loop and doing wide writes if the compiler // doesn't do this for us. - uint32_t first_card_index = first_cluster * ShenandoahCardCluster::CardsPerCluster; + size_t first_card_index = first_cluster * ShenandoahCardCluster::CardsPerCluster; uint8_t *omp = &_overreach_map[first_card_index]; uint8_t *endp = omp + count * ShenandoahCardCluster::CardsPerCluster; while (omp < endp) *omp++ = CardTable::clean_card_val(); } -void ShenandoahDirectCardMarkRememberedSet::merge_overreach(uint32_t first_cluster, uint32_t count) { +void ShenandoahDirectCardMarkRememberedSet::merge_overreach(size_t first_cluster, size_t count) { // We can make this run faster in the future by explicitly unrolling the loop and doing wide writes if the compiler // doesn't do this for us. - uint32_t first_card_index = first_cluster * ShenandoahCardCluster::CardsPerCluster; + size_t first_card_index = first_cluster * ShenandoahCardCluster::CardsPerCluster; uint8_t *bmp = &_byte_map[first_card_index]; uint8_t *endp = bmp + count * ShenandoahCardCluster::CardsPerCluster; uint8_t *omp = &_overreach_map[first_card_index]; @@ -81,3 +83,52 @@ void ShenandoahDirectCardMarkRememberedSet::merge_overreach(uint32_t first_clust while (bmp < endp) *bmp++ &= *omp++; } + +ShenandoahScanRememberedTask::ShenandoahScanRememberedTask(ShenandoahObjToScanQueueSet* queue_set, + ShenandoahObjToScanQueueSet* old_queue_set, + ShenandoahReferenceProcessor* rp, + ShenandoahRegionIterator* regions) : + AbstractGangTask("Scan Remembered Set"), + _queue_set(queue_set), _old_queue_set(old_queue_set), _rp(rp), _regions(regions) {} + +void ShenandoahScanRememberedTask::work(uint worker_id) { + // This sets up a thread local reference to the worker_id which is necessary + // the weak reference processor. + ShenandoahParallelWorkerSession worker_session(worker_id); + + ShenandoahObjToScanQueue* q = _queue_set->queue(worker_id); + ShenandoahObjToScanQueue* old = _old_queue_set == NULL ? NULL : _old_queue_set->queue(worker_id); + ShenandoahMarkRefsClosure cl(q, _rp, old); + RememberedScanner *rs = ShenandoahHeap::heap()->card_scan(); + + // set up thread local closure for shen ref processor + _rp->set_mark_closure(worker_id, &cl); + + ShenandoahHeapRegion* region = _regions->next(); + while (region != NULL) { + if (region->affiliation() == OLD_GENERATION) { + HeapWord *start_of_range = region->bottom(); + size_t start_cluster_no = rs->cluster_for_addr(start_of_range); + + // region->end() represents the end of memory spanned by this region, but not all of this + // memory is eligible to be scanned because some of this memory has not yet been allocated. + // + // region->top() represents the end of allocated memory within this region. Any addresses + // beyond region->top() should not be scanned as that memory does not hold valid objects. + HeapWord *end_of_range = region->top(); + + // end_of_range may point to the middle of a cluster because region->top() may be different than region->end. + // We want to assure that our process_clusters() request spans all relevant clusters. Note that each cluster + // processed will avoid processing beyond end_of_range. + + size_t num_heapwords = end_of_range - start_of_range; + unsigned int cluster_size = CardTable::card_size_in_words * + ShenandoahCardCluster::CardsPerCluster; + size_t num_clusters = (size_t) ((num_heapwords - 1 + cluster_size) / cluster_size); + + // Remembered set scanner + rs->process_clusters(start_cluster_no, num_clusters, end_of_range, &cl); + } + region = _regions->next(); + } +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp index b59f0e62a3135..978d8dffd254d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp @@ -23,14 +23,12 @@ * */ - #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHSCANREMEMBERED_HPP #define SHARE_GC_SHENANDOAH_SHENANDOAHSCANREMEMBERED_HPP -#include -#include "memory/iterator.hpp" -#include "gc/shenandoah/shenandoahCardTable.hpp" - +// During development of this new feature, we want the option to test +// with and without, and to compare performance before and after. +#define FAST_REMEMBERED_SET_SCANNING // Terminology used within this source file: // @@ -119,7 +117,7 @@ // clusters. Initialization cost is essentially identical for // each cluster.) // -// Next, we repeat the process for invocations of process_Clusters. +// Next, we repeat the process for invocations of process_clusters. // for each ShenandoahHeapRegion old_region in the whole heap // determine the cluster number of the first cluster belonging // to that region @@ -211,9 +209,17 @@ // These limitations will be addressed in future enhancements to the // existing implementation. +#include +#include "memory/iterator.hpp" +#include "gc/shared/workgroup.hpp" +#include "gc/shenandoah/shenandoahCardTable.hpp" +#include "gc/shenandoah/shenandoahTaskqueue.hpp" + class ShenandoahReferenceProcessor; class ShenandoahConcurrentMark; class ShenandoahHeap; +class ShenandoahRegionIterator; + class CardTable; class ShenandoahDirectCardMarkRememberedSet: public CHeapObj { @@ -230,9 +236,9 @@ class ShenandoahDirectCardMarkRememberedSet: public CHeapObj { ShenandoahHeap *_heap; CardTable *_card_table; - uint32_t _card_shift; + size_t _card_shift; size_t _total_card_count; - uint32_t _cluster_count; + size_t _cluster_count; HeapWord *_whole_heap_base; // Points to first HeapWord of data contained within heap memory HeapWord *_whole_heap_end; uint8_t *_byte_map; // Points to first entry within the card table @@ -246,25 +252,30 @@ class ShenandoahDirectCardMarkRememberedSet: public CHeapObj { ~ShenandoahDirectCardMarkRememberedSet(); // Card index is zero-based relative to _byte_map. - uint32_t card_index_for_addr(HeapWord *p); - HeapWord *addr_for_card_index(uint32_t card_index); - bool is_card_dirty(uint32_t card_index); - void mark_card_as_dirty(uint32_t card_index); - void mark_card_as_clean(uint32_t card_index); - void mark_overreach_card_as_dirty(uint32_t card_index); + size_t total_cards(); + size_t card_index_for_addr(HeapWord *p); + HeapWord *addr_for_card_index(size_t card_index); + bool is_card_dirty(size_t card_index); + void mark_card_as_dirty(size_t card_index); + void mark_range_as_dirty(size_t card_index, size_t num_cards); + void mark_card_as_clean(size_t card_index); + void mark_range_as_clean(size_t card_index, size_t num_cards); + void mark_overreach_card_as_dirty(size_t card_index); bool is_card_dirty(HeapWord *p); void mark_card_as_dirty(HeapWord *p); + void mark_range_as_dirty(HeapWord *p, size_t num_heap_words); void mark_card_as_clean(HeapWord *p); + void mark_range_as_clean(HeapWord *p, size_t num_heap_words); void mark_overreach_card_as_dirty(void *p); - uint32_t cluster_count(); + size_t cluster_count(); // Called by multiple GC threads at start of concurrent mark and evacuation phases. Each parallel GC thread typically // initializes a different subranges of all overreach entries. - void initialize_overreach(uint32_t first_cluster, uint32_t count); + void initialize_overreach(size_t first_cluster, size_t count); // Called by GC thread at end of concurrent mark or evacuation phase. Each parallel GC thread typically merges different // subranges of all overreach entries. - void merge_overreach(uint32_t first_cluster, uint32_t count); + void merge_overreach(size_t first_cluster, size_t count); }; // A ShenandoahCardCluster represents the minimal unit of work @@ -458,13 +469,65 @@ class ShenandoahCardCluster: public CHeapObj { RememberedSet *_rs; public: - static const uint32_t CardsPerCluster = 64; + static const size_t CardsPerCluster = 64; + +#ifdef FAST_REMEMBERED_SET_SCANNING +private: + // This bit is set iff at least one object starts within a + // particular card region. + static const uint16_t ObjectStartsInCardRegion = 0x8000; + static const uint16_t FirstStartBits = 0x003f; + static const uint16_t LastStartBits = 0x0fc0; + static const uint16_t FirstStartShift = 0; + static const uint16_t LastStartShift = 6; + + static const uint16_t CardOffsetMultiplier = 8; + + uint16_t *object_starts; + +public: + inline void set_first_start(size_t card_index, uint8_t value) { + object_starts[card_index] &= ~FirstStartBits; + object_starts[card_index] |= (FirstStartBits & (value << FirstStartShift)); + } + + inline void set_last_start(size_t card_index, uint8_t value) { + object_starts[card_index] &= ~LastStartBits; + object_starts[card_index] |= (LastStartBits & (value << LastStartShift)); + } + + inline void set_has_object_bit(size_t card_index) { + object_starts[card_index] |= ObjectStartsInCardRegion; + } + + inline void clear_has_object_bit(size_t card_index) { + object_starts[card_index] &= ~ObjectStartsInCardRegion; + } + + inline void clear_objects_in_range(HeapWord *addr, size_t num_words) { + size_t card_index = _rs->card_index_for_addr(addr); + size_t last_card_index = _rs->card_index_for_addr(addr + num_words - 1); + while (card_index <= last_card_index) + object_starts[card_index++] = 0; + } +#endif // FAST_REMEMBERED_SET_SCANNING ShenandoahCardCluster(RememberedSet *rs) { _rs = rs; +#ifdef FAST_REMEMBERED_SET_SCANNING + // HEY! We don't really need object_starts entries for every card entry. We only need these for the + // the card entries that correspond to old-gen memory. But for now, let's be quick and dirty. + object_starts = (uint16_t *) malloc(rs->total_cards() * sizeof(uint16_t)); + if (object_starts == NULL) + fatal("Insufficient memory for initializing heap"); + for (size_t i = 0; i < rs->total_cards(); i++) + object_starts[i] = 0; +#endif } ~ShenandoahCardCluster() { + if (object_starts != NULL) + free(object_starts); } // There is one entry within the object_starts array for each @@ -479,6 +542,7 @@ class ShenandoahCardCluster: public CHeapObj { // within this card region begins. // Bits 0x8000: This bit is on if an object starts within this card // region. + // Bits 0x7000: Reserved for future uses // // In the most recent implementation of ShenandoahScanRemembered::process_clusters(), // there is no need for the get_crossing_object_start() method function, so there is no @@ -548,44 +612,22 @@ class ShenandoahCardCluster: public CHeapObj { // in registering an object. private: - const uint32_t CardByteOffsetMultiplier = 8; - const uint32_t CardWordOffsetMultiplier = 1; - -#ifdef IMPLEMENT_THIS_OPTIMIZATION_LATER - - // This bit is set iff at least one object starts within a - // particular card region. - const uint_16 ObjectStartsInCardRegion = 0x8000; - const uint_16 FirstStartBits = 0x003f; - const uint_16 LastStartBits = 0x0fc0; - const uint_16 FirstStartShift = 0; - const uint_16 LastStartShift = 6; - const uint_16 CrossingObjectOverflow = 0x7fff; - - uint_16 *object_starts; + const size_t CardByteOffsetMultiplier = 8; + const size_t CardWordOffsetMultiplier = 1; public: - inline void set_first_start(uint32_t card_index, uint8_t value) { - object_starts[card_index] &= ~FirstStartBits; - object_starts[card_index] |= (FirstStartBits & (value << FirstStartShift)); - } - - inline void set_last_start(uint32_t card_index, uint8_t value) { - object_starts[card_index] &= ~LastStartBits; - object_starts[card_index] |= (LastStartBits & (value << LastStartShift)); - } +#ifdef CROSSING_OFFSETS_NO_LONGER_NEEDED +private: + const uint16_t CrossingObjectOverflow = 0x7fff; - inline void set_has_object_bit(uint32_t card_index) { - object_starts[card_index] |= ObjectStartsInCardRegion; - } +public: // This has side effect of clearing ObjectStartsInCardRegion bit. - inline void set_crossing_object_start(uint32_t card_index, uint_16 crossing_offset) { + inline void set_crossing_object_start(size_t card_index, uint16_t crossing_offset) { object_starts[card_index] = crossing_offset; } -#endif // IMPLEMENT_THIS_OPTIMIZATION_LATER +#endif // CROSSING_OFFSETS_NO_LONGER_NEEDED -public: // The starting locations of objects contained within old-gen memory // are registered as part of the remembered set implementation. This @@ -678,102 +720,59 @@ class ShenandoahCardCluster: public CHeapObj { // commit-write barriers to assure that access to the // object_starts information is coherent. - static void register_object(void *address, uint32_t length_in_bytes) { - -#ifdef IMPLEMENT_THIS_OPTIMIZATION_LATER - uint32_t card_at_start = cardAtAddress(address); - // The end of this object is contained within this card - uint32_t card_at_end = cardAtAddress(address + length_in_bytes); - - uint32_t cluster_at_start = clusterAtCard(card_at_start); - uint32_t cluster_at_end = clusterAtCard(card_at_end); - - void *card_start_address = addressAtCard(card_at_start); - void *cluster_start_address = addressAtCluster(cluster_at_start); - - uint8_t offset_in_card = - (address - card_start_address) / CardOffsetMultiplier; - - if (!getHashObjectBit(card_at_start)) { - set_has_object_bit(card_at_start); - set_first_start(card_at_start, offset_in_card); - set_last_start(card_at_start, offset_in_card); - } else { - if (offset_in_card < get_first_start(card_at_start)) - set_first_start(card_at_start, offset_in_card); - if (offset_in_card > get_last_start(card_at_start)) - set_last_start(card_at_last, offset_in_card); - } - -#ifdef SUPPORT_FOR_GET_CROSSING_OBJECT_START_NO_LONGER_REQUIRED - - // What is the last card within the current cluster? - uint32_t next_cluster = cluster_at_start + 1; - uint32_t card_at_next_cluster = cardAtCluster(next_cluster); - uint32_t last_card_of_this_cluster = card_at_next_cluster - 1; - uint32_t last_card_in_cluster = ((card_at_end < last_card_of_this_cluster) ? card_at_end: last_card_of_this_cluster); - - uint_16 crossing_map_within_cluster = address - cluster_at_start; - - for (uint32_t card_to_update = card_at_start + 1; card_to_update < last_card_in_cluster; card_to_update++) - set_crossing_object_start(card_to_update, crossing_map_within_cluster); - - // If the last card region of this cluster is completely spanned - // by this new object, set its crossing object start as well. - if (cluster_at_end > cluster_at_start) { - set_crossing_object_start(card_to_update++, crossing_map_within_cluster); - - // Now, we have to update all spanned cards that reside within the - // following clusters. - while (card_to_update < card_at_end) - set_crossing_object_start(card_to_update++, CrossingObjectOverflow); - - // Cases: - // - // 1. The region for card_at_end is not spanned at all because the - // end of the new object aligns with the start of the last - // card region. In this case, we do nothing with the - // card_at_end entry of the object_starts array. - // - // 2. The region for card_at_end is spanned partially. In this - // case, we do nothing with the card_at_end entry of the - // object_starts array. - // - // 3. Do not need to consider the case that the region for - // card_at_end is spanned entirely. If the last region - // spanned by a newly registered object is spanned in its - // entirety, then card_at_end will identify the card region - // that follows the object rather than the last region spanned - // by the object. - - // Bottom line: no further work to be done in any of these cases. - } - // Otherwise, the last card region of this cluster is not - // completely spanned by this new object. Leave the object_starts - // array entry alone. Two cases: - // - // a) This newly registered object represents a consolidation of - // multiple smaller objects, not including the object that - // follows the consolidation. - // b) This newly registered object is created by splitting a - // previously existing object. In this case, the other - // objects resulting from the split will be registered - // separately before it is necessary to lookup any information - // within the object_starts array. - // - -#endif // SUPPORT_FOR_GET_CROSSING_OBJECT_START_NO_LONGER_REQUIRED - -#else // IMPLEMENT_THIS_OPTIMIZATION_LATER - - // Do nothing for now as we have a brute-force implementation - // of findSpanningObject(). - -#endif // IMPLEMENT_THIS_OPTIMIZATION_LATER - } + + // Synchronization thoughts from kelvin: + // + // previously, I had contemplated a more complex implementation of + // object registration, which had to touch every card spanned by the + // registered object. But the current implementation is much simpler, + // and only has to touch the card that contains the start of the + // object. + + // if I were careful to assure that every GCLAB aligns with the start + // of a card region and spanned a multiple of the card region size, + // then the post-processing of each GCLAB with regards to + // register_object() invocations can proceed without synchronization. + + // But right now, we're not even using GCLABs We are doing shared + // allocations. But, we must hold a lock while we are doing these, so + // maybe I just piggy back on the lock that we already hold for + // managing the free lists and register each object newly allocated by + // the shared allocator. + void register_object(HeapWord* address); + + // During the reference updates phase of GC, we walk through each old-gen memory region that was + // not part of the collection set and we invalidate all unmarked objects. As part of this effort, + // we coalesce neighboring dead objects in order to make future remembered set scanning more + // efficient (since future remembered set scanning of any card region containing consecutive + // dead objects can skip over all of them at once by reading only a single dead object header + // instead of having to read the header of each of the coalesced dead objects. + // + // At some future time, we may implement a further optimization: satisfy future allocation requests + // by carving new objects out of the range of memory that represents the coalesced dead objects. + // + // In its current implementation, unregister_object() serves the needs of coalescing objects. + // + + // Suppose we want to combine several dead objects into a single coalesced object. How does this + // impact our representation of crossing map information? + // 1. If the newly coalesced region is contained entirely within a single region, that region's last + // start entry either remains the same or it is changed to the start of the coalesced region. + // 2. For the region that holds the start of the coalesced object, it will not impact the first start + // but it may impact the last start. + // 3. For following regions spanned entirely by the newly coalesced object, it will change has_object + // to false (and make first-start and last-start "undefined"). + // 4. For a following region that is spanned patially by the newly coalesced object, it may change + // first-start value, but it will not change the last-start value. + // + // The range of addresses represented by the arguments to coalesce_objects() must represent a range + // of memory that was previously occupied exactly by one or more previously registered objects. For + // convenience, it is legal to invoke coalesce_objects() with arguments that span a single previously + // registered object. + void coalesce_objects(HeapWord* address, size_t length_in_words); // The typical use case is going to look something like this: - // for each heapregion that comprises olg-gen memory + // for each heapregion that comprises old-gen memory // for each card number that corresponds to this heap region // scan the objects contained therein if the card is dirty // To avoid excessive lookups in a sparse array, the API queries @@ -784,36 +783,18 @@ class ShenandoahCardCluster: public CHeapObj { // associated with addr p. // Returns true iff an object is known to start within the card memory // associated with addr p. - bool has_object(uint32_t card_index); + bool has_object(size_t card_index); // If has_object(card_index), this returns the word offset within this card - // memory at which the first object begins. - uint32_t get_first_start(uint32_t card_index); + // memory at which the first object begins. If !has_object(card_index), the + // result is a don't care value. + size_t get_first_start(size_t card_index); // If has_object(card_index), this returns the word offset within this card - // memory at which the last object begins. - uint32_t get_last_start(uint32_t card_index); + // memory at which the last object begins. If !has_object(card_index), the + // result is a don't care value. + size_t get_last_start(size_t card_index); - // If !has_object(card_index), this returns the offset within the - // enclosing cluster at which the object spanning the start of this - // card memory begins, or returns 0x7fff if the spanning object - // starts before the enclosing cluster. - uint32_t get_crossing_object_start(uint32_t card_index); - - // Card index is zero-based relative to first spanned card region. - uint32_t card_index_for_addr(HeapWord *p); - HeapWord *addr_for_card_index(uint32_t card_index); - bool is_card_dirty(uint32_t card_index); - void mark_card_as_dirty(uint32_t card_index); - void mark_card_as_clean(uint32_t card_index); - void mark_overreach_card_as_dirty(uint32_t card_index); - bool is_card_dirty(HeapWord *p); - void mark_card_as_dirty(HeapWord *p); - void mark_card_as_clean(HeapWord *p); - void mark_overreach_card_as_dirty(void *p); - uint32_t cluster_count(); - void initialize_overreach(uint32_t first_cluster, uint32_t count); - void merge_overreach(uint32_t first_cluster, uint32_t count); }; // ShenandoahScanRemembered is a concrete class representing the @@ -845,7 +826,8 @@ class ShenandoahScanRemembered: public CHeapObj { private: - ShenandoahCardCluster *_scc; + RememberedSet* _rs; + ShenandoahCardCluster* _scc; public: // How to instantiate this object? @@ -863,8 +845,55 @@ class ShenandoahScanRemembered: public CHeapObj { // ShenandoahScanRememberd(rs); - ShenandoahScanRemembered(RememberedSet *rs); - ~ShenandoahScanRemembered(); + ShenandoahScanRemembered(RememberedSet *rs) { + _rs = rs; + _scc = new ShenandoahCardCluster(rs); + } + + ~ShenandoahScanRemembered() { + delete _scc; + } + + // HEY! We really don't want to share all of these APIs with arbitrary consumers of the ShenandoahScanRemembered abstraction. + // But in the spirit of quick and dirty for the time being, I'm going to go ahead and publish everything for right now. Some + // of existing code already depends on having access to these services (because existing code has not been written to honor + // full abstraction of remembered set scanning. In the not too distant future, we want to try to make most, if not all, of + // these services private. Two problems with publicizing: + // 1. Allowing arbitrary users to reach beneath the hood allows the users to make assumptions about underlying implementation. + // This will make it more difficult to change underlying implementation at a future time, such as when we eventually experiment + // with SATB-based implementation of remembered set representation. + // 2. If we carefully control sharing of certain of these services, we can reduce the overhead of synchronization by assuring + // that all users follow protocols that avoid contention that might require synchronization. When we publish these APIs, we + // lose control over who and how the data is accessed. As a result, we are required to insert more defensive measures into + // the implementation, including synchronization locks. + + + // Card index is zero-based relative to first spanned card region. + size_t total_cards(); + size_t card_index_for_addr(HeapWord *p); + HeapWord *addr_for_card_index(size_t card_index); + bool is_card_dirty(size_t card_index); + void mark_card_as_dirty(size_t card_index); + void mark_range_as_dirty(size_t card_index, size_t num_cards); + void mark_card_as_clean(size_t card_index); + void mark_range_as_clean(size_t card_index, size_t num_cards); + void mark_overreach_card_as_dirty(size_t card_index); + bool is_card_dirty(HeapWord *p); + void mark_card_as_dirty(HeapWord *p); + void mark_range_as_dirty(HeapWord *p, size_t num_heap_words); + void mark_card_as_clean(HeapWord *p); + void mark_range_as_clean(HeapWord *p, size_t num_heap_words); + void mark_overreach_card_as_dirty(void *p); + size_t cluster_count(); + void initialize_overreach(size_t first_cluster, size_t count); + void merge_overreach(size_t first_cluster, size_t count); + + size_t cluster_for_addr(HeapWord *addr); + void register_object(HeapWord *addr); + void coalesce_objects(HeapWord *addr, size_t length_in_words); + + // clear the cards to clean, and clear the object_starts info to no objects + void mark_range_as_empty(HeapWord *addr, size_t length_in_words); // process_clusters() scans a portion of the remembered set during a JVM // safepoint as part of the root scanning activities that serve to @@ -898,10 +927,8 @@ class ShenandoahScanRemembered: public CHeapObj { // the template expansions were making it difficult for the link/loader to resolve references to the template- // parameterized implementations of this service. template - void process_clusters(uint worker_id, ShenandoahReferenceProcessor* rp, ShenandoahConcurrentMark* cm, uint32_t first_cluster, uint32_t count, - HeapWord *end_of_range, ClosureType *oops); + inline void process_clusters(size_t first_cluster, size_t count, HeapWord *end_of_range, ClosureType *oops); - uint32_t cluster_for_addr(HeapWord *addr); // To Do: // Create subclasses of ShenandoahInitMarkRootsClosure and @@ -929,4 +956,18 @@ class ShenandoahScanRemembered: public CHeapObj { typedef ShenandoahScanRemembered RememberedScanner; +class ShenandoahScanRememberedTask : public AbstractGangTask { + private: + ShenandoahObjToScanQueueSet* _queue_set; + ShenandoahObjToScanQueueSet* _old_queue_set; + ShenandoahReferenceProcessor* _rp; + ShenandoahRegionIterator* _regions; + public: + ShenandoahScanRememberedTask(ShenandoahObjToScanQueueSet* queue_set, + ShenandoahObjToScanQueueSet* old_queue_set, + ShenandoahReferenceProcessor* rp, + ShenandoahRegionIterator* regions); + + void work(uint worker_id); +}; #endif // SHARE_GC_SHENANDOAH_SHENANDOAHSCANREMEMBERED_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp index 35e25d12a4867..a66af55a4365b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp @@ -34,37 +34,55 @@ #include "gc/shenandoah/shenandoahHeapRegion.hpp" #include "gc/shenandoah/shenandoahScanRemembered.hpp" -inline uint32_t +inline size_t +ShenandoahDirectCardMarkRememberedSet::total_cards() { + return _total_card_count; +} + +inline size_t ShenandoahDirectCardMarkRememberedSet::card_index_for_addr(HeapWord *p) { - return (uint32_t) _card_table->index_for(p); + return _card_table->index_for(p); } inline HeapWord * -ShenandoahDirectCardMarkRememberedSet::addr_for_card_index(uint32_t card_index) { +ShenandoahDirectCardMarkRememberedSet::addr_for_card_index(size_t card_index) { return _whole_heap_base + CardTable::card_size_in_words * card_index; } inline bool -ShenandoahDirectCardMarkRememberedSet::is_card_dirty(uint32_t card_index) { +ShenandoahDirectCardMarkRememberedSet::is_card_dirty(size_t card_index) { uint8_t *bp = &_byte_map[card_index]; return (bp[0] == CardTable::dirty_card_val()); } inline void -ShenandoahDirectCardMarkRememberedSet::mark_card_as_dirty(uint32_t card_index) { +ShenandoahDirectCardMarkRememberedSet::mark_card_as_dirty(size_t card_index) { uint8_t *bp = &_byte_map[card_index]; bp[0] = CardTable::dirty_card_val(); } inline void -ShenandoahDirectCardMarkRememberedSet::mark_card_as_clean(uint32_t card_index) { +ShenandoahDirectCardMarkRememberedSet::mark_range_as_dirty(size_t card_index, size_t num_cards) { + uint8_t *bp = &_byte_map[card_index]; + while (num_cards-- > 0) + *bp++ = CardTable::dirty_card_val(); +} + +inline void +ShenandoahDirectCardMarkRememberedSet::mark_card_as_clean(size_t card_index) { uint8_t *bp = &_byte_map[card_index]; bp[0] = CardTable::clean_card_val(); } inline void -ShenandoahDirectCardMarkRememberedSet::mark_overreach_card_as_dirty( - uint32_t card_index) { +ShenandoahDirectCardMarkRememberedSet::mark_range_as_clean(size_t card_index, size_t num_cards) { + uint8_t *bp = &_byte_map[card_index]; + while (num_cards-- > 0) + *bp++ = CardTable::clean_card_val(); +} + +inline void +ShenandoahDirectCardMarkRememberedSet::mark_overreach_card_as_dirty(size_t card_index) { uint8_t *bp = &_overreach_map[card_index]; bp[0] = CardTable::dirty_card_val(); } @@ -81,43 +99,130 @@ ShenandoahDirectCardMarkRememberedSet::mark_card_as_dirty(HeapWord *p) { bp[0] = CardTable::dirty_card_val(); } +inline void +ShenandoahDirectCardMarkRememberedSet::mark_range_as_dirty(HeapWord *p, size_t num_heap_words) { + uint8_t *bp = &_byte_map_base[uintptr_t(p) >> _card_shift]; + uint8_t *end_bp = &_byte_map_base[uintptr_t(p + num_heap_words) >> _card_shift]; + while (bp < end_bp) + *bp++ = CardTable::dirty_card_val(); +} + inline void ShenandoahDirectCardMarkRememberedSet::mark_card_as_clean(HeapWord *p) { uint8_t *bp = &_byte_map_base[uintptr_t(p) >> _card_shift]; bp[0] = CardTable::clean_card_val(); } +inline void +ShenandoahDirectCardMarkRememberedSet::mark_range_as_clean(HeapWord *p, size_t num_heap_words) { + uint8_t *bp = &_byte_map_base[uintptr_t(p) >> _card_shift]; + uint8_t *end_bp = &_byte_map_base[uintptr_t(p + num_heap_words) >> _card_shift]; + while (bp < end_bp) + *bp++ = CardTable::clean_card_val(); +} + inline void ShenandoahDirectCardMarkRememberedSet::mark_overreach_card_as_dirty(void *p) { uint8_t *bp = &_overreach_map_base[uintptr_t(p) >> _card_shift]; bp[0] = CardTable::dirty_card_val(); } -inline uint32_t +inline size_t ShenandoahDirectCardMarkRememberedSet::cluster_count() { return _cluster_count; } - template -inline uint32_t -ShenandoahCardCluster::card_index_for_addr(HeapWord *p) { - return _rs->card_index_for_addr(p); +inline void +ShenandoahCardCluster::register_object(HeapWord* address) { + size_t card_at_start = _rs->card_index_for_addr(address); + HeapWord *card_start_address = _rs->addr_for_card_index(card_at_start); + uint8_t offset_in_card = address - card_start_address; + + if ((object_starts[card_at_start] & ObjectStartsInCardRegion) == 0) { + set_has_object_bit(card_at_start); + set_first_start(card_at_start, offset_in_card); + set_last_start(card_at_start, offset_in_card); + } else { + if (offset_in_card < get_first_start(card_at_start)) + set_first_start(card_at_start, offset_in_card); + if (offset_in_card > get_last_start(card_at_start)) + set_last_start(card_at_start, offset_in_card); + } } template -inline bool -ShenandoahCardCluster::has_object(uint32_t card_index) { +inline void +ShenandoahCardCluster::coalesce_objects(HeapWord* address, size_t length_in_words) { +#ifdef FAST_REMEMBERED_SET_SCANNING + size_t card_at_start = _rs->card_index_for_addr(address); + HeapWord *card_start_address = _rs->addr_for_card_index(card_at_start); + size_t card_at_end = card_at_start + ((address + length_in_words) - card_start_address) / CardTable::card_size_in_words; + + if (card_at_start == card_at_end) { + // No changes to object_starts array. Either: + // get_first_start(card_at_start) returns this coalesced object, + // or it returns an object that precedes the coalesced object. + // get_last_start(card_at_start) returns the object that immediately follows the coalesced object, + // or it returns an object that comes after the object immediately following the coalesced object. + } else { + uint8_t coalesced_offset = static_cast(address - card_start_address); + if (get_last_start(card_at_start) > coalesced_offset) { + // Existing last start is being coalesced, create new last start + set_last_start(card_at_start, coalesced_offset); + } + // otherwise, get_last_start(card_at_start) must equal coalesced_offset + + // All the cards between first and last get cleared. + for (size_t i = card_at_start + 1; i < card_at_end; i++) { + clear_has_object_bit(i); + } + + uint8_t follow_offset = static_cast((address + length_in_words) - _rs->addr_for_card_index(card_at_end)); + if (has_object(card_at_end) && (get_first_start(card_at_end) < follow_offset)) { + // It may be that after coalescing within this last card's memory range, the last card + // no longer holds an object. + if (get_last_start(card_at_end) >= follow_offset) { + set_first_start(card_at_end, follow_offset); + } else { + // last_start is being coalesced so this card no longer has any objects. + clear_has_object_bit(card_at_end); + } + } + // else + // card_at_end did not have an object, so it still does not have an object, or + // card_at_end had an object that starts after the coalesced object, so no changes required for card_at_end + + } +#else // FAST_REMEMBERED_SET_SCANNING + // Do nothing for now as we have a brute-force implementation + // of findSpanningObject(). +#endif // FAST_REMEMBERED_SET_SCANNING +} - HeapWord *addr = _rs->addr_for_card_index(card_index); +template +inline bool +ShenandoahCardCluster::has_object(size_t card_index) { +#ifdef FAST_REMEMBERED_SET_SCANNING + if (object_starts[card_index] & ObjectStartsInCardRegion) + return true; + else + return false; +#else // FAST_REMEMBERED_SET_SCANNING' ShenandoahHeap *heap = ShenandoahHeap::heap(); + HeapWord *addr = _rs->addr_for_card_index(card_index); ShenandoahHeapRegion *region = heap->heap_region_containing(addr); - // Apparently, region->block_start(addr) is not robust to inquiries beyond top() and it crashes. + // region->block_start(addr) is not robust to inquiries beyond top() and it crashes. if (region->top() <= addr) return false; + // region->block_start(addr) is also not robust to inquiries within a humongous continuation region. + // if region is humongous continuation, no object starts within it. + if (region->is_humongous_continuation()) + return false; + HeapWord *obj = region->block_start(addr); // addr is the first address of the card region. @@ -141,11 +246,16 @@ ShenandoahCardCluster::has_object(uint32_t card_index) { else return false; } +#endif // FAST_REMEMBERED_SET_SCANNING' } template -inline uint32_t -ShenandoahCardCluster::get_first_start(uint32_t card_index) { +inline size_t +ShenandoahCardCluster::get_first_start(size_t card_index) { +#ifdef FAST_REMEMBERED_SET_SCANNING + assert(object_starts[card_index] & ObjectStartsInCardRegion, "Can't get first start because no object starts here"); + return (object_starts[card_index] & FirstStartBits) >> FirstStartShift; +#else // FAST_REMEMBERED_SET_SCANNING HeapWord *addr = _rs->addr_for_card_index(card_index); ShenandoahHeap *heap = ShenandoahHeap::heap(); ShenandoahHeapRegion *region = heap->heap_region_containing(addr); @@ -165,11 +275,16 @@ ShenandoahCardCluster::get_first_start(uint32_t card_index) { assert(obj < end_addr, "Object out of range"); return obj - addr; } +#endif // FAST_REMEMBERED_SET_SCANNING } template -inline uint32_t -ShenandoahCardCluster::get_last_start(uint32_t card_index) { +inline size_t +ShenandoahCardCluster::get_last_start(size_t card_index) { +#ifdef FAST_REMEMBERED_SET_SCANNING + assert(object_starts[card_index] & ObjectStartsInCardRegion, "Can't get last start because no objects starts here"); + return (object_starts[card_index] & LastStartBits) >> LastStartShift; +#else // FAST_REMEMBERED_SET_SCANNING HeapWord *addr = _rs->addr_for_card_index(card_index); HeapWord *end_addr = addr + CardTable::card_size_in_words; ShenandoahHeap *heap = ShenandoahHeap::heap(); @@ -188,13 +303,15 @@ ShenandoahCardCluster::get_last_start(uint32_t card_index) { } assert(obj >= addr, "Object out of range."); return obj - addr; +#endif // FAST_REMEMBERED_SET_SCANNING } +#ifdef CROSSING_OFFSETS_NO_LONGER_NEEDED template -inline uint32_t -ShenandoahCardCluster::get_crossing_object_start(uint32_t card_index) { +inline size_t +ShenandoahCardCluster::get_crossing_object_start(size_t card_index) { HeapWord *addr = _rs->addr_for_card_index(card_index); - uint32_t cluster_no = card_index / ShenandoahCardCluster::CardsPerCluster; + size_t cluster_no = card_index / ShenandoahCardCluster::CardsPerCluster; HeapWord *cluster_addr = _rs->addr_for_card_index(cluster_no * CardsPerCluster); ShenandoahHeap *heap = ShenandoahHeap::heap(); @@ -206,115 +323,127 @@ ShenandoahCardCluster::get_crossing_object_start(uint32_t card_in else return 0x7fff; } +#endif + +template +inline size_t +ShenandoahScanRemembered::total_cards() { return _rs->total_cards(); } + +template +inline size_t +ShenandoahScanRemembered::card_index_for_addr(HeapWord *p) { return _rs->card_index_for_addr(p); }; template inline HeapWord * -ShenandoahCardCluster::addr_for_card_index(uint32_t card_index) { - return _rs->addr_for_card_index(card_index); -} +ShenandoahScanRemembered::addr_for_card_index(size_t card_index) { return _rs->addr_for_card_index(card_index); } template inline bool -ShenandoahCardCluster::is_card_dirty(uint32_t card_index) { - return _rs->is_card_dirty(card_index); -} +ShenandoahScanRemembered::is_card_dirty(size_t card_index) { return _rs->is_card_dirty(card_index); } template inline void -ShenandoahCardCluster::mark_card_as_dirty(uint32_t card_index) { - return _rs->mark_card_as_dirty(card_index); -} +ShenandoahScanRemembered::mark_card_as_dirty(size_t card_index) { _rs->mark_card_as_dirty(card_index); } template inline void -ShenandoahCardCluster::mark_card_as_clean(uint32_t card_index) { - return _rs->mark_card_as_clean(card_index); -} +ShenandoahScanRemembered::mark_range_as_dirty(size_t card_index, size_t num_cards) { _rs->mark_range_as_dirty(card_index, num_cards); } template inline void -ShenandoahCardCluster::mark_overreach_card_as_dirty(uint32_t card_index) { - return _rs->mark_overreach_card_as_dirty(card_index); -} +ShenandoahScanRemembered::mark_card_as_clean(size_t card_index) { _rs->mark_card_as_clean(card_index); } + +template +inline void +ShenandoahScanRemembered::mark_range_as_clean(size_t card_index, size_t num_cards) { _rs->mark_range_as_clean(card_index, num_cards); } + +template +inline void +ShenandoahScanRemembered:: mark_overreach_card_as_dirty(size_t card_index) { _rs->mark_overreach_card_as_dirty(card_index); } template inline bool -ShenandoahCardCluster::is_card_dirty(HeapWord *p) { - return _rs->is_card_dirty(p); -} +ShenandoahScanRemembered::is_card_dirty(HeapWord *p) { return _rs->is_card_dirty(p); } template inline void -ShenandoahCardCluster::mark_card_as_dirty(HeapWord *p) { - return _rs->mark_card_as_dirty(p); -} +ShenandoahScanRemembered::mark_card_as_dirty(HeapWord *p) { _rs->mark_card_as_dirty(p); } template inline void -ShenandoahCardCluster::mark_card_as_clean(HeapWord *p) { - return _rs->mark_card_as_clean(p); -} +ShenandoahScanRemembered::mark_range_as_dirty(HeapWord *p, size_t num_heap_words) { _rs->mark_range_as_dirty(p, num_heap_words); } template inline void -ShenandoahCardCluster::mark_overreach_card_as_dirty(void *p) { - return _rs->mark_overreach_card_as_dirty(p); -} +ShenandoahScanRemembered::mark_card_as_clean(HeapWord *p) { _rs->mark_card_as_clean(p); } template -inline uint32_t -ShenandoahCardCluster::cluster_count() { - return _rs->cluster_count(); -} +inline void +ShenandoahScanRemembered:: mark_range_as_clean(HeapWord *p, size_t num_heap_words) { _rs->mark_range_as_clean(p, num_heap_words); } template inline void -ShenandoahCardCluster::initialize_overreach(uint32_t first_cluster, uint32_t count) { - return _rs->initialize_overreach(first_cluster, count); -} +ShenandoahScanRemembered::mark_overreach_card_as_dirty(void *p) { _rs->mark_overreach_card_as_dirty(p); } + +template +inline size_t +ShenandoahScanRemembered::cluster_count() { return _rs->cluster_count(); } + +template +inline void +ShenandoahScanRemembered::initialize_overreach(size_t first_cluster, size_t count) { _rs->initialize_overreach(first_cluster, count); } + +template +inline void +ShenandoahScanRemembered::merge_overreach(size_t first_cluster, size_t count) { _rs->merge_overreach(first_cluster, count); } + template inline void -ShenandoahCardCluster::merge_overreach(uint32_t first_cluster, uint32_t count) { - return _rs->merge_overreach(first_cluster, count); +ShenandoahScanRemembered::register_object(HeapWord *addr) { + _scc->register_object(addr); } template -ShenandoahScanRemembered::ShenandoahScanRemembered(RememberedSet *rs) { - _scc = new ShenandoahCardCluster(rs); +inline void +ShenandoahScanRemembered::coalesce_objects(HeapWord *addr, size_t length_in_words) { + _scc->coalesce_objects(addr, length_in_words); } template -ShenandoahScanRemembered::~ShenandoahScanRemembered() { - delete _scc; +inline void +ShenandoahScanRemembered::mark_range_as_empty(HeapWord *addr, size_t length_in_words) { + _rs->mark_range_as_clean(addr, length_in_words); + _scc->clear_objects_in_range(addr, length_in_words); } template template inline void -ShenandoahScanRemembered::process_clusters(uint worker_id, ShenandoahReferenceProcessor* rp, ShenandoahConcurrentMark* cm, - uint32_t first_cluster, uint32_t count, HeapWord *end_of_range, - ClosureType *oops) { +ShenandoahScanRemembered::process_clusters(size_t first_cluster, size_t count, HeapWord *end_of_range, + ClosureType *cl) { // Unlike traditional Shenandoah marking, the old-gen resident objects that are examined as part of the remembered set are not // themselves marked. Each such object will be scanned only once. Any young-gen objects referenced from the remembered set will // be marked and then subsequently scanned. while (count-- > 0) { - uint32_t card_index = first_cluster * ShenandoahCardCluster::CardsPerCluster; - uint32_t end_card_index = card_index + ShenandoahCardCluster::CardsPerCluster; + size_t card_index = first_cluster * ShenandoahCardCluster::CardsPerCluster; + size_t end_card_index = card_index + ShenandoahCardCluster::CardsPerCluster; first_cluster++; - int next_card_index = 0; + size_t next_card_index = 0; while (card_index < end_card_index) { - int is_dirty = _scc->is_card_dirty(card_index); - int has_object = _scc->has_object(card_index); + + bool is_dirty = _rs->is_card_dirty(card_index); + bool has_object = _scc->has_object(card_index); if (is_dirty) { if (has_object) { // Scan all objects that start within this card region. - uint32_t start_offset = _scc->get_first_start(card_index); - HeapWord *p = _scc->addr_for_card_index(card_index); + size_t start_offset = _scc->get_first_start(card_index); + HeapWord *p = _rs->addr_for_card_index(card_index); + HeapWord *card_start = p; HeapWord *endp = p + CardTable::card_size_in_words; if (endp > end_of_range) { endp = end_of_range; @@ -322,61 +451,82 @@ ShenandoahScanRemembered::process_clusters(uint worker_id, Shenan } else { // endp either points to start of next card region, or to the next object that needs to be scanned, which may // reside in some successor card region. - next_card_index = _scc->card_index_for_addr(endp); + + // Can't use _scc->card_index_for_addr(endp) here because it crashes with assertion + // failure if endp points to end of heap. + next_card_index = card_index + (endp - card_start) / CardTable::card_size_in_words; } p += start_offset; while (p < endp) { oop obj = oop(p); + // Future TODO: // For improved efficiency, we might want to give special handling of obj->is_objArray(). In // particular, in that case, we might want to divide the effort for scanning of a very long object array // between multiple threads. if (obj->is_objArray()) { - ShenandoahObjToScanQueue* q = cm->get_queue(worker_id); - ShenandoahMarkRefsClosure cl(q, rp); objArrayOop array = objArrayOop(obj); int len = array->length(); - array->oop_iterate_range(&cl, 0, len); + array->oop_iterate_range(cl, 0, len); + } else if (obj->is_instance()) { + obj->oop_iterate(cl); } else { - oops->do_oop(&obj); + // Case 3: Primitive array. Do nothing, no oops there. We use the same + // performance tweak TypeArrayKlass::oop_oop_iterate_impl is using: + // We skip iterating over the klass pointer since we know that + // Universe::TypeArrayKlass never moves. + assert (obj->is_typeArray(), "should be type array"); } p += obj->size(); } - card_index = next_card_index; + if (p > endp) + card_index = card_index + (p - card_start) / CardTable::card_size_in_words; + else // p == endp + card_index = next_card_index; } else { // otherwise, this card will have been scanned during scan of a previous cluster. card_index++; } - } else if (_scc->has_object(card_index)) { + } else if (has_object) { + // Scan the last object that starts within this card memory if it spans at least one dirty card within this cluster // or if it reaches into the next cluster. - uint32_t start_offset = _scc->get_last_start(card_index); - HeapWord *p = _scc->addr_for_card_index(card_index) + start_offset; + size_t start_offset = _scc->get_last_start(card_index); + HeapWord *card_start = _rs->addr_for_card_index(card_index); + HeapWord *p = card_start + start_offset; oop obj = oop(p); HeapWord *nextp = p + obj->size(); - uint32_t last_card = _scc->card_index_for_addr(nextp); + + // Can't use _scc->card_index_for_addr(endp) here because it crashes with assertion + // failure if nextp points to end of heap. + size_t last_card = card_index + (nextp - card_start) / CardTable::card_size_in_words; bool reaches_next_cluster = (last_card > end_card_index); bool spans_dirty_within_this_cluster = false; if (!reaches_next_cluster) { - uint32_t span_card; - for (span_card = card_index+1; span_card < end_card_index; span_card++) - if (_scc->is_card_dirty(span_card)) { + size_t span_card; + for (span_card = card_index+1; span_card <= last_card; span_card++) + if (_rs->is_card_dirty(span_card)) { spans_dirty_within_this_cluster = true; break; } } + if (reaches_next_cluster || spans_dirty_within_this_cluster) { if (obj->is_objArray()) { - ShenandoahObjToScanQueue* q = cm->get_queue(worker_id); // kelvin to confirm: get_queue wants worker_id - ShenandoahMarkRefsClosure cl(q, rp); objArrayOop array = objArrayOop(obj); int len = array->length(); - array->oop_iterate_range(&cl, 0, len); + array->oop_iterate_range(cl, 0, len); + } else if (obj->is_instance()) { + obj->oop_iterate(cl); } else { - oops->do_oop(&obj); + // Case 3: Primitive array. Do nothing, no oops there. We use the same + // performance tweak TypeArrayKlass::oop_oop_iterate_impl is using: + // We skip iterating over the klass pointer since we know that + // Universe::TypeArrayKlass never moves. + assert (obj->is_typeArray(), "should be type array"); } } // Increment card_index to account for the spanning object, even if we didn't scan it. @@ -389,10 +539,10 @@ ShenandoahScanRemembered::process_clusters(uint worker_id, Shenan } template -inline uint32_t +inline size_t ShenandoahScanRemembered::cluster_for_addr(HeapWordImpl **addr) { - uint32_t card_index = _scc->card_index_for_addr(addr); - uint32_t result = card_index / ShenandoahCardCluster::CardsPerCluster; + size_t card_index = _rs->card_index_for_addr(addr); + size_t result = card_index / ShenandoahCardCluster::CardsPerCluster; return result; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahStackWatermark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahStackWatermark.cpp index a0c80d6e72dd7..7d92ba5a49348 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahStackWatermark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahStackWatermark.cpp @@ -75,11 +75,11 @@ OopClosure* ShenandoahStackWatermark::closure_from_context(void* context) { assert(Thread::current()->is_Worker_thread(), "Unexpected thread passing in context: " PTR_FORMAT, p2i(context)); return reinterpret_cast(context); } else { - if (_heap->is_concurrent_mark_in_progress()) { - return &_keep_alive_cl; - } else if (_heap->is_concurrent_weak_root_in_progress()) { + if (_heap->is_concurrent_weak_root_in_progress()) { assert(_heap->is_evacuation_in_progress(), "Nothing to evacuate"); return &_evac_update_oop_cl; + } else if (_heap->is_concurrent_mark_in_progress()) { + return &_keep_alive_cl; } else { ShouldNotReachHere(); return NULL; @@ -92,14 +92,7 @@ void ShenandoahStackWatermark::start_processing_impl(void* context) { ShenandoahHeap* const heap = ShenandoahHeap::heap(); // Process the non-frame part of the thread - if (heap->is_concurrent_mark_in_progress()) { - // We need to reset all TLABs because they might be below the TAMS, and we need to mark - // the objects in them. Do not let mutators allocate any new objects in their current TLABs. - // It is also a good place to resize the TLAB sizes for future allocations. - retire_tlab(); - - _jt->oops_do_no_frames(closure_from_context(context), &_cb_cl); - } else if (heap->is_concurrent_weak_root_in_progress()) { + if (heap->is_concurrent_weak_root_in_progress()) { assert(heap->is_evacuation_in_progress(), "Should not be armed"); // Retire the TLABs, which will force threads to reacquire their TLABs. // This is needed for two reasons. Strong one: new allocations would be with new freeset, @@ -108,6 +101,13 @@ void ShenandoahStackWatermark::start_processing_impl(void* context) { // be needed for reference updates (would update the large filler instead). retire_tlab(); + _jt->oops_do_no_frames(closure_from_context(context), &_cb_cl); + } else if (heap->is_concurrent_mark_in_progress()) { + // We need to reset all TLABs because they might be below the TAMS, and we need to mark + // the objects in them. Do not let mutators allocate any new objects in their current TLABs. + // It is also a good place to resize the TLAB sizes for future allocations. + retire_tlab(); + _jt->oops_do_no_frames(closure_from_context(context), &_cb_cl); } else { ShouldNotReachHere(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahUnload.cpp b/src/hotspot/share/gc/shenandoah/shenandoahUnload.cpp index e916690ec647d..3295c4be27fef 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahUnload.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahUnload.cpp @@ -50,7 +50,7 @@ class ShenandoahIsUnloadingOopClosure : public OopClosure { public: ShenandoahIsUnloadingOopClosure() : - _marking_context(ShenandoahHeap::heap()->complete_marking_context()), + _marking_context(ShenandoahHeap::heap()->marking_context()), _is_unloading(false) { } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp b/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp index 9a7213241b764..869a1e255169e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp @@ -31,6 +31,7 @@ #include "gc/shared/referenceProcessorStats.hpp" #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" #include "gc/shenandoah/shenandoahCollectorPolicy.hpp" +#include "gc/shenandoah/shenandoahGeneration.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahReferenceProcessor.hpp" #include "gc/shenandoah/shenandoahUtils.hpp" @@ -38,19 +39,21 @@ ShenandoahPhaseTimings::Phase ShenandoahTimingsTracker::_current_phase = ShenandoahPhaseTimings::_invalid_phase; -ShenandoahGCSession::ShenandoahGCSession(GCCause::Cause cause) : +ShenandoahGCSession::ShenandoahGCSession(GCCause::Cause cause, ShenandoahGeneration* generation) : _heap(ShenandoahHeap::heap()), + _generation(generation), _timer(_heap->gc_timer()), _tracer(_heap->tracer()) { assert(!ShenandoahGCPhase::is_current_phase_valid(), "No current GC phase"); _heap->set_gc_cause(cause); + _heap->set_gc_generation(generation); _timer->register_gc_start(); _tracer->report_gc_start(cause, _timer->gc_start()); _heap->trace_heap_before_gc(_tracer); _heap->shenandoah_policy()->record_cycle_start(); - _heap->heuristics()->record_cycle_start(); + generation->heuristics()->record_cycle_start(); _trace_cycle.initialize(_heap->cycle_memory_manager(), cause, /* allMemoryPoolsAffected */ true, /* recordGCBeginTime = */ true, @@ -64,7 +67,7 @@ ShenandoahGCSession::ShenandoahGCSession(GCCause::Cause cause) : } ShenandoahGCSession::~ShenandoahGCSession() { - _heap->heuristics()->record_cycle_end(); + _generation->heuristics()->record_cycle_end(); _timer->register_gc_end(); _heap->trace_heap_after_gc(_tracer); _tracer->report_gc_reference_stats(_heap->ref_processor()->reference_process_stats()); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp b/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp index 20ff50002468c..943ef5aa7c635 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp @@ -40,16 +40,18 @@ #include "services/memoryService.hpp" class GCTimer; +class ShenandoahGeneration; class ShenandoahGCSession : public StackObj { private: ShenandoahHeap* const _heap; + ShenandoahGeneration* const _generation; GCTimer* const _timer; GCTracer* const _tracer; TraceMemoryManagerStats _trace_cycle; public: - ShenandoahGCSession(GCCause::Cause cause); + ShenandoahGCSession(GCCause::Cause cause, ShenandoahGeneration* generation); ~ShenandoahGCSession(); }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp index b7ab52e2bbb01..ca159f5871ff3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp @@ -163,7 +163,8 @@ class ShenandoahVerifyOopClosure : public BasicOopIterateClosure { Atomic::add(&_ld[obj_reg->index()], (uint) obj->size(), memory_order_relaxed); // fallthrough for fast failure for un-live regions: case ShenandoahVerifier::_verify_liveness_conservative: - check(ShenandoahAsserts::_safe_oop, obj, obj_reg->has_live(), + check(ShenandoahAsserts::_safe_oop, obj, obj_reg->has_live() || + (obj_reg->is_old() && ShenandoahHeap::heap()->is_gc_generation_young()), "Object must belong to region with live data"); break; default: @@ -218,11 +219,11 @@ class ShenandoahVerifyOopClosure : public BasicOopIterateClosure { // skip break; case ShenandoahVerifier::_verify_marked_incomplete: - check(ShenandoahAsserts::_safe_all, obj, _heap->marking_context()->is_marked(obj), + check(ShenandoahAsserts::_safe_all, obj, _heap->marking_context()->is_marked_or_old(obj), "Must be marked in incomplete bitmap"); break; case ShenandoahVerifier::_verify_marked_complete: - check(ShenandoahAsserts::_safe_all, obj, _heap->complete_marking_context()->is_marked(obj), + check(ShenandoahAsserts::_safe_all, obj, _heap->complete_marking_context()->is_marked_or_old(obj), "Must be marked in complete bitmap"); break; case ShenandoahVerifier::_verify_marked_complete_except_references: @@ -608,7 +609,7 @@ class VerifyThreadGCState : public ThreadClosure { VerifyThreadGCState(const char* label, char expected) : _label(label), _expected(expected) {} void do_thread(Thread* t) { char actual = ShenandoahThreadLocalData::gc_state(t); - if (actual != _expected) { + if (actual != _expected && !(actual & ShenandoahHeap::OLD_MARKING)) { fatal("%s: Thread %s: expected gc-state %d, actual %d", _label, t->name(), _expected, actual); } } @@ -655,7 +656,8 @@ void ShenandoahVerifier::verify_at_safepoint(const char *label, if (enabled) { char actual = _heap->gc_state(); - if (actual != expected) { + // Old generation marking is allowed in all states. + if (actual != expected && !(actual & ShenandoahHeap::OLD_MARKING)) { fatal("%s: Global gc-state: expected %d, actual %d", label, expected, actual); } @@ -742,6 +744,10 @@ void ShenandoahVerifier::verify_at_safepoint(const char *label, if (ShenandoahVerifyLevel >= 4 && marked == _verify_marked_complete && liveness == _verify_liveness_complete) { for (size_t i = 0; i < _heap->num_regions(); i++) { ShenandoahHeapRegion* r = _heap->get_region(i); + if (r->is_old() && _heap->is_gc_generation_young()) { + // Old regions don't have computed live data during young collections. + continue; + } juint verf_live = 0; if (r->is_humongous()) { @@ -954,7 +960,7 @@ class ShenandoahVerifyInToSpaceClosure : public OopClosure { oop obj = CompressedOops::decode_not_null(o); ShenandoahHeap* heap = ShenandoahHeap::heap(); - if (!heap->marking_context()->is_marked(obj)) { + if (!heap->marking_context()->is_marked_or_old(obj)) { ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, NULL, "Verify Roots In To-Space", "Should be marked", __FILE__, __LINE__); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp new file mode 100644 index 0000000000000..f7634d0822e0b --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2020, Red Hat, Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ +#include "precompiled.hpp" + +#include "gc/shenandoah/shenandoahFreeSet.hpp" +#include "gc/shenandoah/shenandoahHeap.hpp" +#include "gc/shenandoah/shenandoahHeap.inline.hpp" +#include "gc/shenandoah/shenandoahUtils.hpp" +#include "gc/shenandoah/shenandoahVerifier.hpp" +#include "gc/shenandoah/shenandoahYoungGeneration.hpp" +#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" + +#undef TRACE_PROMOTION + +ShenandoahYoungGeneration::ShenandoahYoungGeneration(uint max_queues, size_t max_capacity, size_t soft_max_capacity) : + ShenandoahGeneration(YOUNG, max_queues, max_capacity, soft_max_capacity), + _old_gen_task_queues(nullptr) { +} + +const char* ShenandoahYoungGeneration::name() const { + return "YOUNG"; +} + +void ShenandoahYoungGeneration::set_concurrent_mark_in_progress(bool in_progress) { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + heap->set_concurrent_young_mark_in_progress(in_progress); + if (_old_gen_task_queues != nullptr && in_progress) { + // This is not a bug. When the young generation marking is complete, + // the old generation marking is still in progress. + heap->set_concurrent_old_mark_in_progress(in_progress); + } +} + +class ShenandoahPromoteTenuredRegionsTask : public AbstractGangTask { +private: + ShenandoahRegionIterator* _regions; +public: + volatile size_t _used; + volatile size_t _promoted; + + ShenandoahPromoteTenuredRegionsTask(ShenandoahRegionIterator* regions) : + AbstractGangTask("Shenandoah Promote Tenured Regions"), + _regions(regions), + _used(0), + _promoted(0) { + } + + void work(uint worker_id) { + ShenandoahParallelWorkerSession worker_session(worker_id); + ShenandoahHeapRegion* r = _regions->next(); + while (r != NULL) { + if (r->is_young()) { + if (r->age() >= InitialTenuringThreshold && !r->is_humongous_continuation()) { + r->promote(); + Atomic::inc(&_promoted); + } else { + Atomic::add(&_used, r->used()); + } + } + r = _regions->next(); + } + } +}; + +void ShenandoahYoungGeneration::promote_tenured_regions() { + ShenandoahRegionIterator regions; + ShenandoahPromoteTenuredRegionsTask task(®ions); + ShenandoahHeap::heap()->workers()->run_task(&task); + _used = task._used; + log_info(gc)("Promoted " SIZE_FORMAT " regions.", task._promoted); +} + +void ShenandoahYoungGeneration::promote_all_regions() { + // This only happens on a full stw collect. No allocations can happen here. + shenandoah_assert_safepoint(); + + ShenandoahHeap* heap = ShenandoahHeap::heap(); + for (size_t index = 0; index < heap->num_regions(); index++) { + ShenandoahHeapRegion* r = heap->get_region(index); + if (r->is_young()) { + r->promote(); + } + } + assert(_affiliated_region_count == 0, "young generation must not have affiliated regions after reset"); + _used = 0; + + // HEY! Better to use a service of ShenandoahScanRemembered for the following. + + // We can clear the entire card table here because we've just promoted all + // young regions to old, so there can be no old->young pointers at this point. + ShenandoahBarrierSet::barrier_set()->card_table()->clear(); +} + +bool ShenandoahYoungGeneration::contains(ShenandoahHeapRegion* region) const { + return region->affiliation() != OLD_GENERATION; +} + +void ShenandoahYoungGeneration::parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl) { + if (_old_gen_task_queues != NULL) { + // No generation filter on regions, we need to iterate all the regions. + ShenandoahHeap::heap()->parallel_heap_region_iterate(cl); + } else { + // Just the young generations here. + ShenandoahGenerationRegionClosure young_regions(cl); + ShenandoahHeap::heap()->parallel_heap_region_iterate(&young_regions); + } +} + +bool ShenandoahYoungGeneration::is_concurrent_mark_in_progress() { + return ShenandoahHeap::heap()->is_concurrent_young_mark_in_progress(); +} + +void ShenandoahYoungGeneration::reserve_task_queues(uint workers) { + ShenandoahGeneration::reserve_task_queues(workers); + if (_old_gen_task_queues != NULL) { + _old_gen_task_queues->reserve(workers); + } +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp index 588c630d7f5fc..a8b68b00823c0 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, Amazon.com, Inc. and/or its affiliates. All rights reserved. + * Copyright (c) 2021, Amazon.com, Inc. or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -28,8 +28,34 @@ #include "gc/shenandoah/shenandoahGeneration.hpp" class ShenandoahYoungGeneration : public ShenandoahGeneration { +private: + ShenandoahObjToScanQueueSet* _old_gen_task_queues; + public: - ShenandoahYoungGeneration() : ShenandoahGeneration(YOUNG) { } + ShenandoahYoungGeneration(uint max_queues, size_t max_capacity, size_t max_soft_capacity); + + virtual const char* name() const; + + virtual void set_concurrent_mark_in_progress(bool in_progress); + virtual void parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl); + + bool contains(ShenandoahHeapRegion* region) const; + + void promote_tenured_regions(); + void promote_all_regions(); + + void set_old_gen_task_queues(ShenandoahObjToScanQueueSet* old_gen_queues) { + _old_gen_task_queues = old_gen_queues; + } + + ShenandoahObjToScanQueueSet* old_gen_task_queues() const { + return _old_gen_task_queues; + } + + virtual void reserve_task_queues(uint workers); + + protected: + bool is_concurrent_mark_in_progress(); }; #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHYOUNGGENERATION_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp index b1037a83d7bb8..4babedb7cc2fc 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp @@ -361,8 +361,23 @@ \ product(bool, ShenandoahSelfFixing, true, DIAGNOSTIC, \ "Fix references with load reference barrier. Disabling this " \ - "might degrade performance.") - + "might degrade performance.") \ + \ + product(bool, ShenandoahUseSimpleCardScanning, false, DIAGNOSTIC, \ + "Testing: use simplified, very inefficient but much less complex" \ + " card table scanning.") \ + \ + product(bool, ShenandoahPromoteTenuredObjects, true, DIAGNOSTIC, \ + "Turn on/off evacuating individual tenured young objects " \ + " to the old generation.") \ + \ + product(bool, ShenandoahPromoteTenuredRegions, true, DIAGNOSTIC, \ + "Turn on/off transitioning tenured young regions " \ + " to the old generation.") \ + \ + product(bool, ShenandoahAllowOldMarkingPreemption, true, DIAGNOSTIC, \ + "Allow young generation collections to suspend concurrent" \ + " marking in the old generation.") // end of GC_SHENANDOAH_FLAGS #endif // SHARE_GC_SHENANDOAH_SHENANDOAH_GLOBALS_HPP From f3cc407284fab319293e0670eb101084fa36551e Mon Sep 17 00:00:00 2001 From: William Kemper Date: Thu, 11 Mar 2021 17:29:46 +0000 Subject: [PATCH 023/254] Account for usage of all regions in humongous object during promotion Reviewed-by: rkennke --- src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index 6061050f28154..7c8687df7c1b2 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -824,9 +824,6 @@ void ShenandoahHeapRegion::promote() { assert(heap->active_generation()->is_mark_complete(), "sanity"); assert(affiliation() == YOUNG_GENERATION, "Only young regions can be promoted"); - heap->young_generation()->decrease_used(used()); - heap->old_generation()->increase_used(used()); - UpdateCardValuesClosure update_card_values; if (is_humongous_start()) { oop obj = oop(bottom()); @@ -841,6 +838,7 @@ void ShenandoahHeapRegion::promote() { ShenandoahBarrierSet::barrier_set()->card_table()->clear_MemRegion(MemRegion(r->bottom(), r->end())); r->set_affiliation(OLD_GENERATION); + heap->old_generation()->increase_used(used()); } // HEY! Better to call ShenandoahHeap::heap()->card_scan()->mark_range_as_clean(r->bottom(), obj->size()) // and skip the calls to clear_MemRegion() above. @@ -864,6 +862,7 @@ void ShenandoahHeapRegion::promote() { // HEY! Better to call ShenandoahHeap::heap()->card_scan()->mark_range_as_dirty(r->bottom(), obj->size()); ShenandoahBarrierSet::barrier_set()->card_table()->dirty_MemRegion(MemRegion(bottom(), end())); set_affiliation(OLD_GENERATION); + heap->old_generation()->increase_used(used()); oop_iterate_objects(&update_card_values, /*fill_dead_objects*/ true, /* reregister_coalesced_objects */ false); } } From b5575d8e32c17aa94c27aa3e0f50cac8125be9e5 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Tue, 16 Mar 2021 12:03:18 +0000 Subject: [PATCH 024/254] Update periodic gc test for changed heuristic log message Reviewed-by: rkennke --- .../gc/shenandoah/shenandoahRegulatorThread.cpp | 16 ++++++++++------ .../jtreg/gc/shenandoah/TestPeriodicGC.java | 4 ++-- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.cpp index 5f8af48bc0f30..a80a890e4f1e1 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.cpp @@ -91,10 +91,12 @@ void ShenandoahRegulatorThread::regulate_interleaved_cycles() { assert(_global_heuristics != NULL, "Need global heuristics."); while (!should_terminate()) { - if (start_global_cycle()) { - log_info(gc)("Heuristics request for global collection accepted."); - } else if (start_young_cycle()) { - log_info(gc)("Heuristics request for young collection accepted."); + if (_control_thread->gc_mode() == ShenandoahControlThread::none) { + if (start_global_cycle()) { + log_info(gc)("Heuristics request for global collection accepted."); + } else if (start_young_cycle()) { + log_info(gc)("Heuristics request for young collection accepted."); + } } regulator_sleep(); @@ -105,8 +107,10 @@ void ShenandoahRegulatorThread::regulate_heap() { assert(_global_heuristics != NULL, "Need global heuristics."); while (!should_terminate()) { - if (start_global_cycle()) { - log_info(gc)("Heuristics request for global collection accepted."); + if (_control_thread->gc_mode() == ShenandoahControlThread::none) { + if (start_global_cycle()) { + log_info(gc)("Heuristics request for global collection accepted."); + } } regulator_sleep(); diff --git a/test/hotspot/jtreg/gc/shenandoah/TestPeriodicGC.java b/test/hotspot/jtreg/gc/shenandoah/TestPeriodicGC.java index fb6597f412a46..dc199ea687b87 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestPeriodicGC.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestPeriodicGC.java @@ -46,10 +46,10 @@ public static void testWith(String msg, boolean periodic, String... args) throws OutputAnalyzer output = new OutputAnalyzer(pb.start()); output.shouldHaveExitValue(0); - if (periodic && !output.getOutput().contains("Trigger: Time since last GC")) { + if (periodic && !output.getOutput().contains("Trigger (GLOBAL): Time since last GC")) { throw new AssertionError(msg + ": Should have periodic GC in logs"); } - if (!periodic && output.getOutput().contains("Trigger: Time since last GC")) { + if (!periodic && output.getOutput().contains("Trigger (GLOBAL): Time since last GC")) { throw new AssertionError(msg + ": Should not have periodic GC in logs"); } } From 1c4b4ef9c72bbfed7e61735303dab77767099000 Mon Sep 17 00:00:00 2001 From: Bernd Mathiske Date: Thu, 18 Mar 2021 10:56:29 +0000 Subject: [PATCH 025/254] Only scan dirty cards, not entire regions, in update refs remset scan. Reviewed-by: rkennke --- .../share/gc/shenandoah/shenandoahHeap.cpp | 48 ++++++++----------- .../shenandoah/shenandoahScanRemembered.cpp | 22 +-------- .../shenandoah/shenandoahScanRemembered.hpp | 3 ++ .../shenandoahScanRemembered.inline.hpp | 27 +++++++++++ 4 files changed, 52 insertions(+), 48 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 86b592d97b56c..5df3436078996 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -2039,7 +2039,7 @@ class ShenandoahUpdateHeapRefsTask : public AbstractGangTask { void do_work() { T cl; ShenandoahHeapRegion* r = _regions->next(); - // We update references for global and young collections. + // We update references for global, old, and young collections. assert(_heap->active_generation()->is_mark_complete(), "Expected complete marking"); ShenandoahMarkingContext* const ctx = _heap->marking_context(); @@ -2048,40 +2048,34 @@ class ShenandoahUpdateHeapRefsTask : public AbstractGangTask { assert (update_watermark >= r->bottom(), "sanity"); if (r->is_active() && !r->is_cset()) { - if (r->affiliation() == YOUNG_GENERATION || !_heap->mode()->is_generational()) { + if (!_heap->mode()->is_generational() || r->affiliation() == YOUNG_GENERATION) { _heap->marked_object_oop_iterate(r, &cl, update_watermark); } else { assert(r->affiliation() == OLD_GENERATION, "Should not be updating references on FREE regions"); if (!_heap->is_gc_generation_young()) { - // Old region in a global cycle. + // Old region in an old or global cycle. // We need to make sure that the next cycle does not iterate over dead objects // which haven't had their references updated. r->oop_iterate(&cl, /*fill_dead_objects*/ true, /* reregister_coalesced_objects */ true); - } else if (ShenandoahBarrierSet::barrier_set()->card_table()->is_dirty(MemRegion(r->bottom(), r->top()))) { + } else { // Old region in a young cycle. - if (r->is_humongous()) { - r->oop_iterate_humongous(&cl); - } else { - // We don't have liveness information about this region. - // Therefore we process all objects, rather than just marked ones. - // Otherwise subsequent traversals will encounter stale pointers. - - // HEY! kelvin thinks we don't have to update refs throughout the entire region r. We only need - // to update refs for objects that span dirty cards. The code in process clusters does that. We cannot invoke process_clusters - // because that's designed to process transitive closure of live objects. Here, we are just looking - // one level deep in each of the relevant regions. But we can copy and paste some of the code from - // there. - - // HEY moreover! Need to figure out how regions are partitioned between worker threads. Is it possible - // that each region is being processed redundantly by each worker thread? - - HeapWord *p = r->bottom(); - ShenandoahObjectToOopBoundedClosure objs(&cl, p, update_watermark); - // Anything beyond update_watermark is not yet allocated or initialized. - while (p < update_watermark) { - oop obj = oop(p); - objs.do_object(obj); - p += obj->size(); + if (!ShenandoahUseSimpleCardScanning) { + _heap->card_scan()->process_region(r, &cl); + } else if (ShenandoahBarrierSet::barrier_set()->card_table()->is_dirty(MemRegion(r->bottom(), r->top()))) { + if (r->is_humongous()) { + r->oop_iterate_humongous(&cl); + } else { + // We don't have liveness information about this region. + // Therefore we process all objects, rather than just marked ones. + // Otherwise subsequent traversals will encounter stale pointers. + HeapWord *p = r->bottom(); + ShenandoahObjectToOopBoundedClosure objs(&cl, p, update_watermark); + // Anything beyond update_watermark is not yet allocated or initialized. + while (p < update_watermark) { + oop obj = oop(p); + objs.do_object(obj); + p += obj->size(); + } } } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp index c0891aafe6419..e21b1370b38c5 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp @@ -107,27 +107,7 @@ void ShenandoahScanRememberedTask::work(uint worker_id) { ShenandoahHeapRegion* region = _regions->next(); while (region != NULL) { if (region->affiliation() == OLD_GENERATION) { - HeapWord *start_of_range = region->bottom(); - size_t start_cluster_no = rs->cluster_for_addr(start_of_range); - - // region->end() represents the end of memory spanned by this region, but not all of this - // memory is eligible to be scanned because some of this memory has not yet been allocated. - // - // region->top() represents the end of allocated memory within this region. Any addresses - // beyond region->top() should not be scanned as that memory does not hold valid objects. - HeapWord *end_of_range = region->top(); - - // end_of_range may point to the middle of a cluster because region->top() may be different than region->end. - // We want to assure that our process_clusters() request spans all relevant clusters. Note that each cluster - // processed will avoid processing beyond end_of_range. - - size_t num_heapwords = end_of_range - start_of_range; - unsigned int cluster_size = CardTable::card_size_in_words * - ShenandoahCardCluster::CardsPerCluster; - size_t num_clusters = (size_t) ((num_heapwords - 1 + cluster_size) / cluster_size); - - // Remembered set scanner - rs->process_clusters(start_cluster_no, num_clusters, end_of_range, &cl); + rs->process_region(region, &cl); } region = _regions->next(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp index 978d8dffd254d..b3590a6a0f147 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp @@ -213,6 +213,7 @@ #include "memory/iterator.hpp" #include "gc/shared/workgroup.hpp" #include "gc/shenandoah/shenandoahCardTable.hpp" +#include "gc/shenandoah/shenandoahHeapRegion.hpp" #include "gc/shenandoah/shenandoahTaskqueue.hpp" class ShenandoahReferenceProcessor; @@ -929,6 +930,8 @@ class ShenandoahScanRemembered: public CHeapObj { template inline void process_clusters(size_t first_cluster, size_t count, HeapWord *end_of_range, ClosureType *oops); + template + inline void process_region(ShenandoahHeapRegion* region, ClosureType *cl); // To Do: // Create subclasses of ShenandoahInitMarkRootsClosure and diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp index a66af55a4365b..8727e16f7d33f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp @@ -538,6 +538,33 @@ ShenandoahScanRemembered::process_clusters(size_t first_cluster, } } +template +template +inline void +ShenandoahScanRemembered::process_region(ShenandoahHeapRegion *region, ClosureType *cl) { + HeapWord *start_of_range = region->bottom(); + size_t start_cluster_no = cluster_for_addr(start_of_range); + + // region->end() represents the end of memory spanned by this region, but not all of this + // memory is eligible to be scanned because some of this memory has not yet been allocated. + // + // region->top() represents the end of allocated memory within this region. Any addresses + // beyond region->top() should not be scanned as that memory does not hold valid objects. + HeapWord *end_of_range = region->top(); + + // end_of_range may point to the middle of a cluster because region->top() may be different than region->end. + // We want to assure that our process_clusters() request spans all relevant clusters. Note that each cluster + // processed will avoid processing beyond end_of_range. + + size_t num_heapwords = end_of_range - start_of_range; + unsigned int cluster_size = CardTable::card_size_in_words * + ShenandoahCardCluster::CardsPerCluster; + size_t num_clusters = (size_t) ((num_heapwords - 1 + cluster_size) / cluster_size); + + // Remembered set scanner + process_clusters(start_cluster_no, num_clusters, end_of_range, cl); +} + template inline size_t ShenandoahScanRemembered::cluster_for_addr(HeapWordImpl **addr) { From 41ba54b6a123b6abf363c8ee77c29ff23128ebfa Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Thu, 18 Mar 2021 20:03:03 +0000 Subject: [PATCH 026/254] Fix ShenandoahPurgeSATBTask Reviewed-by: rkennke --- .../gc/shenandoah/shenandoahOldGeneration.cpp | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp index f110953f2675a..40378073cf85e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp @@ -38,18 +38,14 @@ class ShenandoahFlushAllSATB : public ThreadClosure { private: SATBMarkQueueSet& _satb_qset; - uintx _claim_token; public: explicit ShenandoahFlushAllSATB(SATBMarkQueueSet& satb_qset) : - _satb_qset(satb_qset), - _claim_token(Threads::thread_claim_token()) {} + _satb_qset(satb_qset) { } void do_thread(Thread* thread) { - if (thread->claim_threads_do(true, _claim_token)) { - // Transfer any partial buffer to the qset for completed buffer processing. - _satb_qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread)); - } + // Transfer any partial buffer to the qset for completed buffer processing. + _satb_qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread)); } }; @@ -92,13 +88,16 @@ class ShenandoahProcessOldSATB : public SATBBufferClosure { }; class ShenandoahPurgeSATBTask : public AbstractGangTask { - private: +private: ShenandoahObjToScanQueueSet* _mark_queues; - public: + +public: size_t _trashed_oops; explicit ShenandoahPurgeSATBTask(ShenandoahObjToScanQueueSet* queues) : - AbstractGangTask("Purge SATB"), _mark_queues(queues), _trashed_oops(0) {} + AbstractGangTask("Purge SATB"), + _mark_queues(queues), + _trashed_oops(0) {} ~ShenandoahPurgeSATBTask() { if (_trashed_oops > 0) { @@ -110,7 +109,7 @@ class ShenandoahPurgeSATBTask : public AbstractGangTask { ShenandoahParallelWorkerSession worker_session(worker_id); ShenandoahSATBMarkQueueSet &satb_queues = ShenandoahBarrierSet::satb_mark_queue_set(); ShenandoahFlushAllSATB flusher(satb_queues); - Threads::threads_do(&flusher); + Threads::possibly_parallel_threads_do(true /*par*/, &flusher); ShenandoahObjToScanQueue* mark_queue = _mark_queues->queue(worker_id); ShenandoahProcessOldSATB processor(mark_queue); From bcde2f616eb006f89aaf2503c862f3bf0f54224f Mon Sep 17 00:00:00 2001 From: William Kemper Date: Fri, 19 Mar 2021 11:26:10 +0000 Subject: [PATCH 027/254] Restore ShenandoahVerify functionality for generational mode Reviewed-by: rkennke --- .../share/gc/shenandoah/shenandoahFreeSet.cpp | 4 +- .../gc/shenandoah/shenandoahGeneration.hpp | 3 + .../shenandoah/shenandoahGlobalGeneration.cpp | 4 + .../shenandoah/shenandoahGlobalGeneration.hpp | 2 + .../gc/shenandoah/shenandoahHeapRegion.cpp | 13 ++- .../gc/shenandoah/shenandoahHeapRegion.hpp | 3 +- .../gc/shenandoah/shenandoahOldGeneration.cpp | 5 + .../gc/shenandoah/shenandoahOldGeneration.hpp | 3 + .../gc/shenandoah/shenandoahRootVerifier.cpp | 17 ++++ .../gc/shenandoah/shenandoahRootVerifier.hpp | 3 +- .../shenandoah/shenandoahScanRemembered.hpp | 1 + .../shenandoahScanRemembered.inline.hpp | 36 +++++++ .../gc/shenandoah/shenandoahVerifier.cpp | 95 +++++++++++++++---- .../gc/shenandoah/shenandoahVerifier.hpp | 2 +- .../shenandoah/shenandoahYoungGeneration.cpp | 17 ++-- .../shenandoah/shenandoahYoungGeneration.hpp | 2 + 16 files changed, 179 insertions(+), 31 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index 8d5df95ec6c63..eadb51ff644d9 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -364,9 +364,9 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req) { increase_used(ShenandoahHeapRegion::region_size_bytes() * num); if (req.affiliation() == ShenandoahRegionAffiliation::YOUNG_GENERATION) { - _heap->young_generation()->increase_used(ShenandoahHeapRegion::region_size_bytes() * num); + _heap->young_generation()->increase_used(words_size * HeapWordSize); } else if (req.affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) { - _heap->old_generation()->increase_used(ShenandoahHeapRegion::region_size_bytes() * num); + _heap->old_generation()->increase_used(words_size * HeapWordSize); } if (remainder != 0) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index bbfdb5becc3b5..bc8cf7b42bfae 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -92,6 +92,9 @@ class ShenandoahGeneration : public CHeapObj { // Apply closure to all regions affiliated with this generation. virtual void parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl) = 0; + // Apply closure to all regions affiliated with this generation (single threaded). + virtual void heap_region_iterate(ShenandoahHeapRegionClosure* cl) = 0; + // This is public to support cancellation of marking when a Full cycle is started. virtual void set_concurrent_mark_in_progress(bool in_progress) = 0; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp index 5869d19109a43..0533275c48d95 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp @@ -78,6 +78,10 @@ void ShenandoahGlobalGeneration::parallel_heap_region_iterate(ShenandoahHeapRegi ShenandoahHeap::heap()->parallel_heap_region_iterate(cl); } +void ShenandoahGlobalGeneration::heap_region_iterate(ShenandoahHeapRegionClosure* cl) { + ShenandoahHeap::heap()->heap_region_iterate(cl); +} + bool ShenandoahGlobalGeneration::is_concurrent_mark_in_progress() { ShenandoahHeap* heap = ShenandoahHeap::heap(); return heap->is_concurrent_mark_in_progress(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp index 4355807bb1145..8cc55a8184521 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp @@ -48,6 +48,8 @@ class ShenandoahGlobalGeneration : public ShenandoahGeneration { void parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl); + void heap_region_iterate(ShenandoahHeapRegionClosure* cl); + protected: bool is_concurrent_mark_in_progress(); }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index 7c8687df7c1b2..fe2c8daa61c38 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -816,7 +816,7 @@ class UpdateCardValuesClosure : public BasicOopIterateClosure { } }; -void ShenandoahHeapRegion::promote() { +size_t ShenandoahHeapRegion::promote() { assert(SafepointSynchronize::is_at_safepoint(), "must be at a safepoint"); ShenandoahHeap* heap = ShenandoahHeap::heap(); @@ -825,6 +825,9 @@ void ShenandoahHeapRegion::promote() { assert(affiliation() == YOUNG_GENERATION, "Only young regions can be promoted"); UpdateCardValuesClosure update_card_values; + ShenandoahGeneration* old_generation = heap->old_generation(); + ShenandoahGeneration* young_generation = heap->young_generation(); + if (is_humongous_start()) { oop obj = oop(bottom()); assert(marking_context->is_marked(obj), "promoted humongous object should be alive"); @@ -838,7 +841,8 @@ void ShenandoahHeapRegion::promote() { ShenandoahBarrierSet::barrier_set()->card_table()->clear_MemRegion(MemRegion(r->bottom(), r->end())); r->set_affiliation(OLD_GENERATION); - heap->old_generation()->increase_used(used()); + old_generation->increase_used(r->used()); + young_generation->decrease_used(r->used()); } // HEY! Better to call ShenandoahHeap::heap()->card_scan()->mark_range_as_clean(r->bottom(), obj->size()) // and skip the calls to clear_MemRegion() above. @@ -846,6 +850,7 @@ void ShenandoahHeapRegion::promote() { // Iterate over all humongous regions that are spanned by the humongous object obj. The remnant // of memory in the last humongous region that is not spanned by obj is currently not used. obj->oop_iterate(&update_card_values); + return index_limit - index(); } else { log_debug(gc)("promoting region " SIZE_FORMAT ", clear cards from " SIZE_FORMAT " to " SIZE_FORMAT, index(), (size_t) bottom(), (size_t) top()); @@ -862,7 +867,9 @@ void ShenandoahHeapRegion::promote() { // HEY! Better to call ShenandoahHeap::heap()->card_scan()->mark_range_as_dirty(r->bottom(), obj->size()); ShenandoahBarrierSet::barrier_set()->card_table()->dirty_MemRegion(MemRegion(bottom(), end())); set_affiliation(OLD_GENERATION); - heap->old_generation()->increase_used(used()); + old_generation->increase_used(used()); + young_generation->decrease_used(used()); oop_iterate_objects(&update_card_values, /*fill_dead_objects*/ true, /* reregister_coalesced_objects */ false); + return 1; } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp index 83455eabbd947..23f882f0ed009 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp @@ -403,7 +403,8 @@ class ShenandoahHeapRegion { void increment_age() { if (_age < markWord::max_age) { _age++; } } void reset_age() { _age = 0; } - void promote(); + // If this is a humongous start, returns the number of regions in the object. + size_t promote(); private: void do_commit(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp index 40378073cf85e..d533e3da7ba5a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp @@ -135,6 +135,11 @@ void ShenandoahOldGeneration::parallel_heap_region_iterate(ShenandoahHeapRegionC ShenandoahHeap::heap()->parallel_heap_region_iterate(&old_regions); } +void ShenandoahOldGeneration::heap_region_iterate(ShenandoahHeapRegionClosure* cl) { + ShenandoahGenerationRegionClosure old_regions(cl); + ShenandoahHeap::heap()->heap_region_iterate(&old_regions); +} + void ShenandoahOldGeneration::set_concurrent_mark_in_progress(bool in_progress) { ShenandoahHeap::heap()->set_concurrent_old_mark_in_progress(in_progress); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp index 70052c7c7bd6d..c6a7bdb9a49b0 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp @@ -38,6 +38,9 @@ class ShenandoahOldGeneration : public ShenandoahGeneration { bool contains(ShenandoahHeapRegion* region) const; void parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl); + + void heap_region_iterate(ShenandoahHeapRegionClosure* cl); + void set_concurrent_mark_in_progress(bool in_progress); // We leave the SATB barrier on for the entirety of the old generation diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.cpp b/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.cpp index 6992279c41e09..60b701a5d2105 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.cpp @@ -32,6 +32,7 @@ #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahPhaseTimings.hpp" #include "gc/shenandoah/shenandoahRootVerifier.hpp" +#include "gc/shenandoah/shenandoahScanRemembered.inline.hpp" #include "gc/shenandoah/shenandoahStringDedup.hpp" #include "gc/shenandoah/shenandoahUtils.hpp" #include "gc/shared/oopStorage.inline.hpp" @@ -109,6 +110,12 @@ void ShenandoahRootVerifier::oops_do(OopClosure* oops) { ShenandoahStringDedup::oops_do_slow(oops); } + ShenandoahHeap* heap = ShenandoahHeap::heap(); + if (heap->mode()->is_generational() && heap->is_gc_generation_young() && verify(RememberedSetRoots)) { + shenandoah_assert_safepoint(); + heap->card_scan()->oops_do(oops); + } + if (verify(ThreadRoots)) { shenandoah_assert_safepoint(); // Do thread roots the last. This allows verification code to find @@ -131,6 +138,11 @@ void ShenandoahRootVerifier::roots_do(OopClosure* oops) { JNIHandles::oops_do(oops); Universe::vm_global()->oops_do(oops); + ShenandoahHeap* heap = ShenandoahHeap::heap(); + if (heap->mode()->is_generational() && heap->is_gc_generation_young()) { + heap->card_scan()->oops_do(oops); + } + // Do thread roots the last. This allows verification code to find // any broken objects from those special roots first, not the accidental // dangling reference from the thread root. @@ -149,6 +161,11 @@ void ShenandoahRootVerifier::strong_roots_do(OopClosure* oops) { JNIHandles::oops_do(oops); Universe::vm_global()->oops_do(oops); + ShenandoahHeap* heap = ShenandoahHeap::heap(); + if (heap->mode()->is_generational() && heap->is_gc_generation_young()) { + heap->card_scan()->oops_do(oops); + } + // Do thread roots the last. This allows verification code to find // any broken objects from those special roots first, not the accidental // dangling reference from the thread root. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.hpp b/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.hpp index d79624f6f37c5..8e5a7f98bd177 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.hpp @@ -50,7 +50,8 @@ class ShenandoahRootVerifier : public StackObj { WeakRoots = 1 << 4, StringDedupRoots = 1 << 5, JNIHandleRoots = 1 << 6, - AllRoots = (SerialRoots | ThreadRoots | CodeRoots | CLDGRoots | WeakRoots | StringDedupRoots | JNIHandleRoots) + RememberedSetRoots = 1 << 7, + AllRoots = (SerialRoots | ThreadRoots | CodeRoots | CLDGRoots | WeakRoots | StringDedupRoots | JNIHandleRoots | RememberedSetRoots) }; private: diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp index b3590a6a0f147..85a9b16ef9d04 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp @@ -955,6 +955,7 @@ class ShenandoahScanRemembered: public CHeapObj { // implementations will want to update this value each time they // cross one of these boundaries. + void oops_do(OopClosure* cl); }; typedef ShenandoahScanRemembered RememberedScanner; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp index 8727e16f7d33f..e3604397250c3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp @@ -573,4 +573,40 @@ ShenandoahScanRemembered::cluster_for_addr(HeapWordImpl **addr) { return result; } +class ShenandoahOopIterateAdapter : public BasicOopIterateClosure { + private: + OopClosure* _cl; + public: + explicit ShenandoahOopIterateAdapter(OopClosure* cl) : _cl(cl) {} + + void do_oop(oop* o) { + _cl->do_oop(o); + } + + void do_oop(narrowOop* o) { + _cl->do_oop(o); + } +}; + +template +inline void ShenandoahScanRemembered::oops_do(OopClosure* cl) { + ShenandoahOopIterateAdapter adapter(cl); + ShenandoahHeap* heap = ShenandoahHeap::heap(); + for (size_t i = 0, n = heap->num_regions(); i < n; ++i) { + ShenandoahHeapRegion* region = heap->get_region(i); + if (region->affiliation() == OLD_GENERATION) { + HeapWord* start_of_range = region->bottom(); + HeapWord* end_of_range = region->top(); + size_t start_cluster_no = cluster_for_addr(start_of_range); + size_t num_heapwords = end_of_range - start_of_range; + unsigned int cluster_size = CardTable::card_size_in_words * + ShenandoahCardCluster::CardsPerCluster; + size_t num_clusters = (size_t) ((num_heapwords - 1 + cluster_size) / cluster_size); + + // Remembered set scanner + process_clusters(start_cluster_no, num_clusters, end_of_range, &adapter); + } + } +} + #endif // SHARE_GC_SHENANDOAH_SHENANDOAHSCANREMEMBEREDINLINE_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp index ca159f5871ff3..8423bd51934f9 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp @@ -27,6 +27,7 @@ #include "gc/shenandoah/shenandoahAsserts.hpp" #include "gc/shenandoah/shenandoahForwarding.inline.hpp" #include "gc/shenandoah/shenandoahPhaseTimings.hpp" +#include "gc/shenandoah/shenandoahGeneration.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" #include "gc/shenandoah/shenandoahRootProcessor.hpp" @@ -67,6 +68,7 @@ class ShenandoahVerifyOopClosure : public BasicOopIterateClosure { ShenandoahLivenessData* _ld; void* _interior_loc; oop _loc; + ShenandoahGeneration* _generation; public: ShenandoahVerifyOopClosure(ShenandoahVerifierStack* stack, MarkBitMap* map, ShenandoahLivenessData* ld, @@ -78,11 +80,17 @@ class ShenandoahVerifyOopClosure : public BasicOopIterateClosure { _map(map), _ld(ld), _interior_loc(NULL), - _loc(NULL) { + _loc(NULL), + _generation(NULL) { if (options._verify_marked == ShenandoahVerifier::_verify_marked_complete_except_references || options._verify_marked == ShenandoahVerifier::_verify_marked_disable) { set_ref_discoverer_internal(new ShenandoahIgnoreReferenceDiscoverer()); } + + if (_heap->mode()->is_generational()) { + _generation = _heap->active_generation(); + assert(_generation != NULL, "Expected active generation in this mode"); + } } private: @@ -106,13 +114,22 @@ class ShenandoahVerifyOopClosure : public BasicOopIterateClosure { // For performance reasons, only fully verify non-marked field values. // We are here when the host object for *p is already marked. - if (_map->par_mark(obj)) { + if ( in_generation(obj) && _map->par_mark(obj)) { verify_oop_at(p, obj); _stack->push(ShenandoahVerifierTask(obj)); } } } + bool in_generation(oop obj) { + if (_generation == NULL) { + return true; + } + + ShenandoahHeapRegion* region = _heap->heap_region_containing(obj); + return _generation->contains(region); + } + void verify_oop(oop obj) { // Perform consistency checks with gradually decreasing safety level. This guarantees // that failure report would not try to touch something that was not yet verified to be @@ -123,7 +140,7 @@ class ShenandoahVerifyOopClosure : public BasicOopIterateClosure { check(ShenandoahAsserts::_safe_unknown, obj, is_object_aligned(obj), "oop must be aligned"); - ShenandoahHeapRegion *obj_reg = _heap->heap_region_containing(obj); + ShenandoahHeapRegion* obj_reg = _heap->heap_region_containing(obj); Klass* obj_klass = obj->klass_or_null(); // Verify that obj is not in dead space: @@ -134,7 +151,7 @@ class ShenandoahVerifyOopClosure : public BasicOopIterateClosure { check(ShenandoahAsserts::_safe_unknown, obj, Metaspace::contains(obj_klass), "Object klass pointer must go to metaspace"); - HeapWord *obj_addr = cast_from_oop(obj); + HeapWord* obj_addr = cast_from_oop(obj); check(ShenandoahAsserts::_safe_unknown, obj, obj_addr < obj_reg->top(), "Object start should be within the region"); @@ -199,7 +216,7 @@ class ShenandoahVerifyOopClosure : public BasicOopIterateClosure { check(ShenandoahAsserts::_safe_oop, obj, !fwd_reg->is_humongous(), "Should have no humongous forwardees"); - HeapWord *fwd_addr = cast_from_oop(fwd); + HeapWord* fwd_addr = cast_from_oop(fwd); check(ShenandoahAsserts::_safe_oop, obj, fwd_addr < fwd_reg->top(), "Forwardee start should be within the region"); check(ShenandoahAsserts::_safe_oop, obj, (fwd_addr + fwd->size()) <= fwd_reg->top(), @@ -213,7 +230,14 @@ class ShenandoahVerifyOopClosure : public BasicOopIterateClosure { } // ------------ obj and fwd are safe at this point -------------- - + // We allow for marked or old here for two reasons: + // 1. If this is a young collect, old objects wouldn't be marked. We've + // recently change the verifier traversal to only follow young objects + // during a young collect so this _shouldn't_ be necessary. + // 2. At present, we do not clear dead objects from the remembered set. + // Everything in the remembered set is old (ipso facto), so allowing for + // 'marked_or_old' covers the case of stale objects in rset. + // TODO: Just use 'is_marked' here. switch (_options._verify_marked) { case ShenandoahVerifier::_verify_marked_disable: // skip @@ -227,7 +251,7 @@ class ShenandoahVerifyOopClosure : public BasicOopIterateClosure { "Must be marked in complete bitmap"); break; case ShenandoahVerifier::_verify_marked_complete_except_references: - check(ShenandoahAsserts::_safe_all, obj, _heap->complete_marking_context()->is_marked(obj), + check(ShenandoahAsserts::_safe_all, obj, _heap->complete_marking_context()->is_marked_or_old(obj), "Must be marked in complete bitmap, except j.l.r.Reference referents"); break; default: @@ -492,11 +516,12 @@ class ShenandoahVerifierMarkedRegionTask : public AbstractGangTask { private: const char* _label; ShenandoahVerifier::VerifyOptions _options; - ShenandoahHeap *_heap; + ShenandoahHeap* _heap; MarkBitMap* _bitmap; ShenandoahLivenessData* _ld; volatile size_t _claimed; volatile size_t _processed; + ShenandoahGeneration* _generation; public: ShenandoahVerifierMarkedRegionTask(MarkBitMap* bitmap, @@ -510,7 +535,13 @@ class ShenandoahVerifierMarkedRegionTask : public AbstractGangTask { _bitmap(bitmap), _ld(ld), _claimed(0), - _processed(0) {}; + _processed(0), + _generation(NULL) { + if (_heap->mode()->is_generational()) { + _generation = _heap->active_generation(); + assert(_generation != NULL, "Expected active generation in this mode."); + } + }; size_t processed() { return Atomic::load(&_processed); @@ -526,6 +557,10 @@ class ShenandoahVerifierMarkedRegionTask : public AbstractGangTask { size_t v = Atomic::fetch_and_add(&_claimed, 1u, memory_order_relaxed); if (v < _heap->num_regions()) { ShenandoahHeapRegion* r = _heap->get_region(v); + if (!in_generation(r)) { + continue; + } + if (!r->is_humongous() && !r->is_trash()) { work_regular(r, stack, cl); } else if (r->is_humongous_start()) { @@ -537,7 +572,11 @@ class ShenandoahVerifierMarkedRegionTask : public AbstractGangTask { } } - virtual void work_humongous(ShenandoahHeapRegion *r, ShenandoahVerifierStack& stack, ShenandoahVerifyOopClosure& cl) { + bool in_generation(ShenandoahHeapRegion* r) { + return _generation == NULL || _generation->contains(r); + } + + virtual void work_humongous(ShenandoahHeapRegion* r, ShenandoahVerifierStack& stack, ShenandoahVerifyOopClosure& cl) { size_t processed = 0; HeapWord* obj = r->bottom(); if (_heap->complete_marking_context()->is_marked((oop)obj)) { @@ -546,7 +585,7 @@ class ShenandoahVerifierMarkedRegionTask : public AbstractGangTask { Atomic::add(&_processed, processed, memory_order_relaxed); } - virtual void work_regular(ShenandoahHeapRegion *r, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl) { + virtual void work_regular(ShenandoahHeapRegion* r, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl) { size_t processed = 0; ShenandoahMarkingContext* ctx = _heap->complete_marking_context(); HeapWord* tams = ctx->top_at_mark_start(r); @@ -579,7 +618,7 @@ class ShenandoahVerifierMarkedRegionTask : public AbstractGangTask { Atomic::add(&_processed, processed, memory_order_relaxed); } - void verify_and_follow(HeapWord *addr, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl, size_t *processed) { + void verify_and_follow(HeapWord* addr, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl, size_t* processed) { if (!_bitmap->par_mark(addr)) return; // Verify the object itself: @@ -615,7 +654,7 @@ class VerifyThreadGCState : public ThreadClosure { } }; -void ShenandoahVerifier::verify_at_safepoint(const char *label, +void ShenandoahVerifier::verify_at_safepoint(const char* label, VerifyForwarded forwarded, VerifyMarked marked, VerifyCollectionSet cset, VerifyLiveness liveness, VerifyRegions regions, @@ -690,10 +729,35 @@ void ShenandoahVerifier::verify_at_safepoint(const char *label, byte_size_in_proper_unit(cl.committed()), proper_unit_for_byte_size(cl.committed())); } + ShenandoahGeneration* generation; + if (_heap->mode()->is_generational()) { + generation = _heap->active_generation(); + guarantee(generation != NULL, "Need to know which generation to verify."); + } else { + generation = NULL; + } + + if (generation != NULL) { + ShenandoahHeapLocker lock(_heap->lock()); + + ShenandoahCalculateRegionStatsClosure cl; + generation->heap_region_iterate(&cl); + size_t generation_used = generation->used(); + guarantee(cl.used() == generation_used, + "%s: generation (%s) used size must be consistent: generation-used = " SIZE_FORMAT "%s, regions-used = " SIZE_FORMAT "%s", + label, generation->name(), + byte_size_in_proper_unit(generation_used), proper_unit_for_byte_size(generation_used), + byte_size_in_proper_unit(cl.used()), proper_unit_for_byte_size(cl.used())); + } + // Internal heap region checks if (ShenandoahVerifyLevel >= 1) { ShenandoahVerifyHeapRegionClosure cl(label, regions); - _heap->heap_region_iterate(&cl); + if (generation != NULL) { + generation->heap_region_iterate(&cl); + } else { + _heap->heap_region_iterate(&cl); + } } OrderAccess::fence(); @@ -744,8 +808,7 @@ void ShenandoahVerifier::verify_at_safepoint(const char *label, if (ShenandoahVerifyLevel >= 4 && marked == _verify_marked_complete && liveness == _verify_liveness_complete) { for (size_t i = 0; i < _heap->num_regions(); i++) { ShenandoahHeapRegion* r = _heap->get_region(i); - if (r->is_old() && _heap->is_gc_generation_young()) { - // Old regions don't have computed live data during young collections. + if (generation != NULL && !generation->contains(r)) { continue; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp index 838daf955b953..a2e9eb357615d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp @@ -166,7 +166,7 @@ class ShenandoahVerifier : public CHeapObj { }; private: - void verify_at_safepoint(const char *label, + void verify_at_safepoint(const char* label, VerifyForwarded forwarded, VerifyMarked marked, VerifyCollectionSet cset, diff --git a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp index f7634d0822e0b..653d011136053 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp @@ -56,13 +56,11 @@ class ShenandoahPromoteTenuredRegionsTask : public AbstractGangTask { private: ShenandoahRegionIterator* _regions; public: - volatile size_t _used; volatile size_t _promoted; ShenandoahPromoteTenuredRegionsTask(ShenandoahRegionIterator* regions) : AbstractGangTask("Shenandoah Promote Tenured Regions"), _regions(regions), - _used(0), _promoted(0) { } @@ -71,11 +69,12 @@ class ShenandoahPromoteTenuredRegionsTask : public AbstractGangTask { ShenandoahHeapRegion* r = _regions->next(); while (r != NULL) { if (r->is_young()) { + // The thread that first encounters a humongous start region is responsible + // for promoting the continuation regions so we need this guard here to + // keep other worker threads from trying to promote the continuations. if (r->age() >= InitialTenuringThreshold && !r->is_humongous_continuation()) { - r->promote(); - Atomic::inc(&_promoted); - } else { - Atomic::add(&_used, r->used()); + size_t promoted = r->promote(); + Atomic::add(&_promoted, promoted); } } r = _regions->next(); @@ -87,7 +86,6 @@ void ShenandoahYoungGeneration::promote_tenured_regions() { ShenandoahRegionIterator regions; ShenandoahPromoteTenuredRegionsTask task(®ions); ShenandoahHeap::heap()->workers()->run_task(&task); - _used = task._used; log_info(gc)("Promoted " SIZE_FORMAT " regions.", task._promoted); } @@ -127,6 +125,11 @@ void ShenandoahYoungGeneration::parallel_heap_region_iterate(ShenandoahHeapRegio } } +void ShenandoahYoungGeneration::heap_region_iterate(ShenandoahHeapRegionClosure* cl) { + ShenandoahGenerationRegionClosure young_regions(cl); + ShenandoahHeap::heap()->heap_region_iterate(&young_regions); +} + bool ShenandoahYoungGeneration::is_concurrent_mark_in_progress() { return ShenandoahHeap::heap()->is_concurrent_young_mark_in_progress(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp index a8b68b00823c0..1f12df47169fa 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp @@ -39,6 +39,8 @@ class ShenandoahYoungGeneration : public ShenandoahGeneration { virtual void set_concurrent_mark_in_progress(bool in_progress); virtual void parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl); + void heap_region_iterate(ShenandoahHeapRegionClosure* cl); + bool contains(ShenandoahHeapRegion* region) const; void promote_tenured_regions(); From d6b1c3f016453b1c449bca0b83c83027ad51e59c Mon Sep 17 00:00:00 2001 From: Aleksey Shipilev Date: Fri, 2 Apr 2021 07:08:05 +0000 Subject: [PATCH 028/254] Clean up CondCardMark code: remove excess barriers, enable it by default Reviewed-by: zgu --- .../shenandoahBarrierSetAssembler_aarch64.cpp | 7 ------- .../shenandoah/shenandoahBarrierSetAssembler_x86.cpp | 3 --- .../share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp | 6 ------ .../share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp | 11 +---------- .../gc/shenandoah/mode/shenandoahGenerationalMode.cpp | 3 +++ .../gc/shenandoah/shenandoahBarrierSet.inline.hpp | 7 +------ 6 files changed, 5 insertions(+), 32 deletions(-) diff --git a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp index ca20eb0eb7973..ee391c86d4714 100644 --- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp @@ -390,15 +390,11 @@ void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register o if (UseCondCardMark) { Label L_already_dirty; - __ membar(Assembler::StoreLoad); __ ldrb(rscratch2, Address(obj, rscratch1)); __ cbz(rscratch2, L_already_dirty); __ strb(zr, Address(obj, rscratch1)); __ bind(L_already_dirty); } else { - // if (ct->scanned_concurrently()) { - // __ membar(Assembler::StoreStore); - // } __ strb(zr, Address(obj, rscratch1)); } } @@ -647,9 +643,6 @@ void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssemb __ load_byte_map_base(scratch); __ add(start, start, scratch); - // if (ct->scanned_concurrently()) { - // __ membar(__ StoreStore); - // } __ bind(L_loop); __ strb(zr, Address(start, count)); __ subs(count, count, 1); diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp index 6014847857ef5..adc0c18f9294a 100644 --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp @@ -656,9 +656,6 @@ void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register o int dirty = CardTable::dirty_card_val(); if (UseCondCardMark) { Label L_already_dirty; -// if (ct->scanned_concurrently()) { -// __ membar(Assembler::StoreLoad); -// } __ cmpb(card_addr, dirty); __ jcc(Assembler::equal, L_already_dirty); __ movb(card_addr, dirty); diff --git a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp index 275924ea0742a..2fdc66acf189d 100644 --- a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp +++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp @@ -353,9 +353,6 @@ void ShenandoahBarrierSetC1::post_barrier(LIRAccess& access, LIR_OprDesc* addr, LIR_Opr dirty = LIR_OprFact::intConst(CardTable::dirty_card_val()); if (UseCondCardMark) { LIR_Opr cur_value = gen->new_register(T_INT); -// if (ct->scanned_concurrently()) { -// __ membar_storeload(); -// } __ move(card_addr, cur_value); LabelObj* L_already_dirty = new LabelObj(); @@ -364,9 +361,6 @@ void ShenandoahBarrierSetC1::post_barrier(LIRAccess& access, LIR_OprDesc* addr, __ move(dirty, card_addr); __ branch_destination(L_already_dirty->label()); } else { -// if (ct->scanned_concurrently()) { -// __ membar_storestore(); -// } __ move(dirty, card_addr); } } diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp index c50a03a35084d..9812c814662ea 100644 --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp @@ -522,10 +522,6 @@ void ShenandoahBarrierSetC2::post_barrier(GraphKit* kit, Node* zero = __ ConI(0); // Dirty card value if (UseCondCardMark) { -// if (ct->scanned_concurrently()) { -// kit->insert_mem_bar(Op_MemBarVolatile, oop_store); -// __ sync_kit(kit); -// } // The classic GC reference write barrier is typically implemented // as a store into the global card mark table. Unfortunately // unconditional stores can result in false sharing and excessive @@ -538,12 +534,7 @@ void ShenandoahBarrierSetC2::post_barrier(GraphKit* kit, } // Smash zero into card -// if(!ct->scanned_concurrently()) { - __ store(__ ctrl(), card_adr, zero, T_BYTE, adr_type, MemNode::unordered); -// } else { -// // Specialized path for CM store barrier -// __ storeCM(__ ctrl(), card_adr, zero, oop_store, adr_idx, T_BYTE, adr_type); -// } + __ store(__ ctrl(), card_adr, zero, T_BYTE, adr_type, MemNode::unordered); if (UseCondCardMark) { __ end_if(); diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp index 33e2a99616932..1430e815b3e3b 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp @@ -47,6 +47,9 @@ void ShenandoahGenerationalMode::initialize_flags() const { SHENANDOAH_ERGO_ENABLE_FLAG(ExplicitGCInvokesConcurrent); SHENANDOAH_ERGO_ENABLE_FLAG(ShenandoahImplicitGCInvokesConcurrent); + // This helps most multi-core hardware hosts, enable by default + SHENANDOAH_ERGO_ENABLE_FLAG(UseCondCardMark); + // Final configuration checks SHENANDOAH_CHECK_FLAG_SET(ShenandoahLoadRefBarrier); SHENANDOAH_CHECK_FLAG_UNSET(ShenandoahIUBarrier); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp index 27ac335f35d26..b92fe6d9407a8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp @@ -196,12 +196,7 @@ template inline void ShenandoahBarrierSet::write_ref_field_post(T* field, oop newVal) { if (ShenandoahHeap::heap()->mode()->is_generational()) { volatile CardTable::CardValue* byte = card_table()->byte_for(field); - // if (card_table()->scanned_concurrently()) { - // // Perform a releasing store if the card table is scanned concurrently - // Atomic::release_store(byte, CardTable::dirty_card_val()); - // } else { - *byte = CardTable::dirty_card_val(); - // } + *byte = CardTable::dirty_card_val(); } } From bad8e744b8b544de757a2c3901a47988e5c74d0f Mon Sep 17 00:00:00 2001 From: William Kemper Date: Mon, 5 Apr 2021 16:50:11 +0000 Subject: [PATCH 029/254] Add parallel worker timings for remembered set scan Reviewed-by: shade --- .../gc/shenandoah/shenandoahGeneration.cpp | 1 + .../gc/shenandoah/shenandoahPhaseTimings.cpp | 1 + .../gc/shenandoah/shenandoahPhaseTimings.hpp | 3 +++ .../gc/shenandoah/shenandoahScanRemembered.cpp | 17 +++++++++-------- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index f6f46f047a58e..86fe4fb3e935c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -242,6 +242,7 @@ void ShenandoahGeneration::scan_remembered_set() { uint nworkers = heap->workers()->active_workers(); reserve_task_queues(nworkers); + ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_scan_rset); ShenandoahReferenceProcessor* rp = heap->ref_processor(); ShenandoahRegionIterator regions; ShenandoahScanRememberedTask task(task_queues(), old_gen_task_queues(), rp, ®ions); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.cpp b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.cpp index 6618559d661b8..dc81575057d0f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.cpp @@ -97,6 +97,7 @@ bool ShenandoahPhaseTimings::is_worker_phase(Phase phase) { assert(phase >= 0 && phase < _num_phases, "Out of bounds"); switch (phase) { case init_evac: + case init_scan_rset: case finish_mark: case purge_weak_par: case full_gc_mark: diff --git a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp index b490de9d05017..1901c7cbdadb0 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp @@ -46,6 +46,7 @@ class outputStream; f(CNT_PREFIX ## StringDedupQueueRoots, DESC_PREFIX "Dedup Queue Roots") \ f(CNT_PREFIX ## WeakRefProc, DESC_PREFIX "Weak References") \ f(CNT_PREFIX ## ParallelMark, DESC_PREFIX "Parallel Mark") \ + f(CNT_PREFIX ## ScanClusters, DESC_PREFIX "Scan Clusters") // end #define SHENANDOAH_PHASE_DO(f) \ @@ -54,6 +55,8 @@ class outputStream; f(init_mark_gross, "Pause Init Mark (G)") \ f(init_mark, "Pause Init Mark (N)") \ f(init_manage_tlabs, " Manage TLABs") \ + f(init_scan_rset, " Scan Remembered Set") \ + SHENANDOAH_PAR_PHASE_DO(init_scan_rset_, " RS: ", f) \ f(init_update_region_states, " Update Region States") \ \ f(conc_mark_roots, "Concurrent Mark Roots ") \ diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp index e21b1370b38c5..b2bfa191619f2 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp @@ -31,7 +31,7 @@ #include "gc/shenandoah/shenandoahReferenceProcessor.hpp" #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp" -ShenandoahDirectCardMarkRememberedSet::ShenandoahDirectCardMarkRememberedSet(CardTable *card_table, size_t total_card_count) { +ShenandoahDirectCardMarkRememberedSet::ShenandoahDirectCardMarkRememberedSet(CardTable* card_table, size_t total_card_count) { _heap = ShenandoahHeap::heap(); _card_table = card_table; _total_card_count = total_card_count; @@ -64,8 +64,8 @@ void ShenandoahDirectCardMarkRememberedSet::initialize_overreach(size_t first_cl // unrolling the loop and doing wide writes if the compiler // doesn't do this for us. size_t first_card_index = first_cluster * ShenandoahCardCluster::CardsPerCluster; - uint8_t *omp = &_overreach_map[first_card_index]; - uint8_t *endp = omp + count * ShenandoahCardCluster::CardsPerCluster; + uint8_t* omp = &_overreach_map[first_card_index]; + uint8_t* endp = omp + count * ShenandoahCardCluster::CardsPerCluster; while (omp < endp) *omp++ = CardTable::clean_card_val(); } @@ -75,9 +75,9 @@ void ShenandoahDirectCardMarkRememberedSet::merge_overreach(size_t first_cluster // We can make this run faster in the future by explicitly unrolling the loop and doing wide writes if the compiler // doesn't do this for us. size_t first_card_index = first_cluster * ShenandoahCardCluster::CardsPerCluster; - uint8_t *bmp = &_byte_map[first_card_index]; - uint8_t *endp = bmp + count * ShenandoahCardCluster::CardsPerCluster; - uint8_t *omp = &_overreach_map[first_card_index]; + uint8_t* bmp = &_byte_map[first_card_index]; + uint8_t* endp = bmp + count * ShenandoahCardCluster::CardsPerCluster; + uint8_t* omp = &_overreach_map[first_card_index]; // dirty_card is 0, clean card is 0xff; if either *bmp or *omp is dirty, we need to mark it as dirty while (bmp < endp) @@ -95,11 +95,12 @@ void ShenandoahScanRememberedTask::work(uint worker_id) { // This sets up a thread local reference to the worker_id which is necessary // the weak reference processor. ShenandoahParallelWorkerSession worker_session(worker_id); + ShenandoahWorkerTimingsTracker x(ShenandoahPhaseTimings::init_scan_rset, ShenandoahPhaseTimings::ScanClusters, worker_id); ShenandoahObjToScanQueue* q = _queue_set->queue(worker_id); ShenandoahObjToScanQueue* old = _old_queue_set == NULL ? NULL : _old_queue_set->queue(worker_id); ShenandoahMarkRefsClosure cl(q, _rp, old); - RememberedScanner *rs = ShenandoahHeap::heap()->card_scan(); + RememberedScanner* scanner = ShenandoahHeap::heap()->card_scan(); // set up thread local closure for shen ref processor _rp->set_mark_closure(worker_id, &cl); @@ -107,7 +108,7 @@ void ShenandoahScanRememberedTask::work(uint worker_id) { ShenandoahHeapRegion* region = _regions->next(); while (region != NULL) { if (region->affiliation() == OLD_GENERATION) { - rs->process_region(region, &cl); + scanner->process_region(region, &cl); } region = _regions->next(); } From 84bb2c3c1229e00dd51e15fb2499305b77b69be4 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Thu, 8 Apr 2021 21:18:54 +0000 Subject: [PATCH 030/254] Use timed wait to sleep control thread Reviewed-by: shade --- .../gc/shenandoah/shenandoahControlThread.cpp | 20 ++++++++++++++----- .../gc/shenandoah/shenandoahControlThread.hpp | 2 ++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index 54de48d8c9a5b..9a8b7373871b8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -52,6 +52,7 @@ ShenandoahControlThread::ShenandoahControlThread() : ConcurrentGCThread(), _alloc_failure_waiters_lock(Mutex::leaf, "ShenandoahAllocFailureGC_lock", true, Monitor::_safepoint_check_always), _gc_waiters_lock(Mutex::leaf, "ShenandoahRequestedGC_lock", true, Monitor::_safepoint_check_always), + _control_lock(Mutex::leaf - 1, "ShenandoahControlGC_lock", true, Monitor::_safepoint_check_never), _periodic_task(this), _requested_gc_cause(GCCause::_no_cause_specified), _requested_generation(GenerationMode::GLOBAL), @@ -367,11 +368,12 @@ void ShenandoahControlThread::run_service() { last_shrink_time = current; } - // HEY! kemperw would like to have this thread sleep on a timed wait so it - // could be explicitly woken when there is something to do. The timed wait - // is necessary because this thread has a responsibility to send - // 'alloc_words' to the pacer when it does not perform a GC. - os::naked_short_sleep(ShenandoahControlIntervalMin); + { + // The timed wait is necessary because this thread has a responsibility to send + // 'alloc_words' to the pacer when it does not perform a GC. + MonitorLocker lock(&_control_lock, Mutex::_no_safepoint_check_flag); + lock.wait(ShenandoahControlIntervalMax); + } } // Wait for the actual stop(), can't leave run_service() earlier. @@ -721,6 +723,7 @@ bool ShenandoahControlThread::request_concurrent_gc(GenerationMode generation) { if (_mode == none) { _requested_gc_cause = GCCause::_shenandoah_concurrent_gc; _requested_generation = generation; + notify_control_thread(); return true; } @@ -730,12 +733,18 @@ bool ShenandoahControlThread::request_concurrent_gc(GenerationMode generation) { _requested_generation = generation; _preemption_requested.set(); ShenandoahHeap::heap()->cancel_gc(GCCause::_shenandoah_concurrent_gc); + notify_control_thread(); return true; } return false; } +void ShenandoahControlThread::notify_control_thread() { + MonitorLocker locker(&_control_lock, Mutex::_no_safepoint_check_flag); + _control_lock.notify(); +} + bool ShenandoahControlThread::preempt_old_marking(GenerationMode generation) { return generation == YOUNG && _allow_old_preemption.is_set(); } @@ -756,6 +765,7 @@ void ShenandoahControlThread::handle_requested_gc(GCCause::Cause cause) { while (current_gc_id < required_gc_id) { _gc_requested.set(); _requested_gc_cause = cause; + notify_control_thread(); ml.wait(); current_gc_id = get_gc_id(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp index 051710b054cf6..6fdc479252147 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp @@ -62,6 +62,7 @@ class ShenandoahControlThread: public ConcurrentGCThread { // to make complete explicit cycle for for demanding customers. Monitor _alloc_failure_waiters_lock; Monitor _gc_waiters_lock; + Monitor _control_lock; ShenandoahPeriodicTask _periodic_task; ShenandoahPeriodicPacerNotify _periodic_pacer_notify_task; @@ -175,6 +176,7 @@ class ShenandoahControlThread: public ConcurrentGCThread { private: static const char* gc_mode_name(GCMode mode); + void notify_control_thread(); }; #endif // SHARE_GC_SHENANDOAH_SHENANDOAHCONTROLTHREAD_HPP From 00aae9c80322595b7bf07e26bd9840f1f3892716 Mon Sep 17 00:00:00 2001 From: Zhengyu Gu Date: Mon, 12 Apr 2021 15:35:35 +0000 Subject: [PATCH 031/254] Refix ShenandoahPurgeSATBTask Reviewed-by: rkennke --- .../gc/shenandoah/shenandoahOldGeneration.cpp | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp index d533e3da7ba5a..8fceb6a2cd167 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp @@ -38,14 +38,18 @@ class ShenandoahFlushAllSATB : public ThreadClosure { private: SATBMarkQueueSet& _satb_qset; + uintx _claim_token; public: explicit ShenandoahFlushAllSATB(SATBMarkQueueSet& satb_qset) : - _satb_qset(satb_qset) { } + _satb_qset(satb_qset), + _claim_token(Threads::thread_claim_token()) { } void do_thread(Thread* thread) { - // Transfer any partial buffer to the qset for completed buffer processing. - _satb_qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread)); + if (thread->claim_threads_do(true, _claim_token)) { + // Transfer any partial buffer to the qset for completed buffer processing. + _satb_qset.flush_queue(ShenandoahThreadLocalData::satb_mark_queue(thread)); + } } }; @@ -92,12 +96,14 @@ class ShenandoahPurgeSATBTask : public AbstractGangTask { ShenandoahObjToScanQueueSet* _mark_queues; public: - size_t _trashed_oops; + volatile size_t _trashed_oops; explicit ShenandoahPurgeSATBTask(ShenandoahObjToScanQueueSet* queues) : AbstractGangTask("Purge SATB"), _mark_queues(queues), - _trashed_oops(0) {} + _trashed_oops(0) { + Threads::change_thread_claim_token(); + } ~ShenandoahPurgeSATBTask() { if (_trashed_oops > 0) { @@ -109,7 +115,7 @@ class ShenandoahPurgeSATBTask : public AbstractGangTask { ShenandoahParallelWorkerSession worker_session(worker_id); ShenandoahSATBMarkQueueSet &satb_queues = ShenandoahBarrierSet::satb_mark_queue_set(); ShenandoahFlushAllSATB flusher(satb_queues); - Threads::possibly_parallel_threads_do(true /*par*/, &flusher); + Threads::threads_do(&flusher); ShenandoahObjToScanQueue* mark_queue = _mark_queues->queue(worker_id); ShenandoahProcessOldSATB processor(mark_queue); From eebec31d08233f6a758b7964255038fb6ca82c02 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Wed, 21 Apr 2021 11:24:47 +0000 Subject: [PATCH 032/254] Update remembered set during old generation mark Reviewed-by: rkennke --- src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp | 1 + src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp | 8 ++++++++ src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp | 5 +++++ 3 files changed, 14 insertions(+) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index 5194432e20366..022087da082c8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -669,6 +669,7 @@ class ShenandoahHeap : public CollectedHeap { public: inline RememberedScanner* card_scan() { return _card_scan; } void clear_cards_for(ShenandoahHeapRegion* region); + void mark_card_as_dirty(HeapWord* location); // ---------- Helper functions // diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp index 647352f3f48c5..21a7fa51b19aa 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp @@ -40,6 +40,7 @@ #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" #include "gc/shenandoah/shenandoahControlThread.hpp" #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp" +#include "gc/shenandoah/shenandoahScanRemembered.inline.hpp" #include "gc/shenandoah/shenandoahThreadLocalData.hpp" #include "gc/shenandoah/mode/shenandoahMode.hpp" #include "oops/compressedOops.inline.hpp" @@ -579,4 +580,11 @@ inline void ShenandoahHeap::clear_cards_for(ShenandoahHeapRegion* region) { } } +inline void ShenandoahHeap::mark_card_as_dirty(HeapWord* location) { + if (mode()->is_generational()) { + _card_scan->mark_card_as_dirty(location); + } +} + + #endif // SHARE_GC_SHENANDOAH_SHENANDOAHHEAP_INLINE_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp index ef5a614bf36a3..5f9205d970927 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp @@ -269,8 +269,13 @@ inline void ShenandoahMark::mark_through_ref(T *p, ShenandoahObjToScanQueue* q, mark_ref(q, mark_context, weak, obj); shenandoah_assert_marked(p, obj); } else if (old != nullptr) { + // Young mark, bootstrapping old. mark_ref(old, mark_context, weak, obj); shenandoah_assert_marked(p, obj); + } else if (GENERATION == OLD) { + // Old mark, found a young pointer. + assert(ShenandoahHeap::heap()->is_in_young(obj), "Expected young object."); + ShenandoahHeap::heap()->mark_card_as_dirty((HeapWord*)p); } } } From 68ce49bd85a0d6a98d4d62a79226e453027977e9 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Tue, 27 Apr 2021 14:03:33 +0000 Subject: [PATCH 033/254] Mixed evacuation Reviewed-by: zgu, rkennke --- .../shenandoahAdaptiveOldHeuristics.cpp | 305 +++++++++++++++++ .../shenandoahAdaptiveOldHeuristics.hpp | 111 ++++++ .../shenandoahAggressiveOldHeuristics.cpp | 69 ++++ .../shenandoahAggressiveOldHeuristics.hpp | 47 +++ .../shenandoahCompactOldHeuristics.cpp | 99 ++++++ .../shenandoahCompactOldHeuristics.hpp | 45 +++ .../heuristics/shenandoahHeuristics.cpp | 39 ++- .../heuristics/shenandoahHeuristics.hpp | 23 +- .../heuristics/shenandoahOldHeuristics.cpp | 317 ++++++++++++++++++ .../heuristics/shenandoahOldHeuristics.hpp | 105 ++++++ .../shenandoahPassiveOldHeuristics.cpp | 75 +++++ .../shenandoahPassiveOldHeuristics.hpp | 50 +++ .../shenandoahStaticOldHeuristics.cpp | 74 ++++ .../shenandoahStaticOldHeuristics.hpp | 47 +++ .../gc/shenandoah/mode/shenandoahMode.cpp | 22 ++ .../gc/shenandoah/mode/shenandoahMode.hpp | 4 +- .../shenandoah/mode/shenandoahPassiveMode.cpp | 6 + .../shenandoah/mode/shenandoahPassiveMode.hpp | 3 +- .../gc/shenandoah/shenandoahConcurrentGC.hpp | 2 + .../gc/shenandoah/shenandoahGeneration.cpp | 21 +- .../gc/shenandoah/shenandoahGeneration.hpp | 11 +- .../share/gc/shenandoah/shenandoahHeap.cpp | 4 +- .../share/gc/shenandoah/shenandoahHeap.hpp | 6 + .../gc/shenandoah/shenandoahHeapRegion.cpp | 74 ++-- .../gc/shenandoah/shenandoahHeapRegion.hpp | 4 + .../share/gc/shenandoah/shenandoahOldGC.cpp | 91 +++++ .../share/gc/shenandoah/shenandoahOldGC.hpp | 4 + .../gc/shenandoah/shenandoahPhaseTimings.hpp | 2 + .../shenandoah/shenandoahRegulatorThread.cpp | 4 +- .../shenandoah/shenandoahRegulatorThread.hpp | 3 +- .../generational/TestConcurrentEvac.java | 209 ++++++++++++ 31 files changed, 1816 insertions(+), 60 deletions(-) create mode 100644 src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveOldHeuristics.cpp create mode 100644 src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveOldHeuristics.hpp create mode 100644 src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveOldHeuristics.cpp create mode 100644 src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveOldHeuristics.hpp create mode 100644 src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactOldHeuristics.cpp create mode 100644 src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactOldHeuristics.hpp create mode 100644 src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp create mode 100644 src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp create mode 100644 src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveOldHeuristics.cpp create mode 100644 src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveOldHeuristics.hpp create mode 100644 src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticOldHeuristics.cpp create mode 100644 src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticOldHeuristics.hpp create mode 100644 test/hotspot/jtreg/gc/shenandoah/generational/TestConcurrentEvac.java diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveOldHeuristics.cpp new file mode 100644 index 0000000000000..66d6e018f0187 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveOldHeuristics.cpp @@ -0,0 +1,305 @@ +/* + * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp" +#include "gc/shenandoah/heuristics/shenandoahAdaptiveOldHeuristics.hpp" +#include "gc/shenandoah/shenandoahCollectionSet.hpp" +#include "gc/shenandoah/shenandoahFreeSet.hpp" +#include "gc/shenandoah/shenandoahGeneration.hpp" +#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" +#include "logging/log.hpp" +#include "logging/logTag.hpp" +#include "utilities/quickSort.hpp" + +// These constants are used to adjust the margin of error for the moving +// average of the allocation rate and cycle time. The units are standard +// deviations. +const double ShenandoahAdaptiveOldHeuristics::FULL_PENALTY_SD = 0.2; +const double ShenandoahAdaptiveOldHeuristics::DEGENERATE_PENALTY_SD = 0.1; + +// These are used to decide if we want to make any adjustments at all +// at the end of a successful concurrent cycle. +const double ShenandoahAdaptiveOldHeuristics::LOWEST_EXPECTED_AVAILABLE_AT_END = -0.5; +const double ShenandoahAdaptiveOldHeuristics::HIGHEST_EXPECTED_AVAILABLE_AT_END = 0.5; + +// These values are the confidence interval expressed as standard deviations. +// At the minimum confidence level, there is a 25% chance that the true value of +// the estimate (average cycle time or allocation rate) is not more than +// MINIMUM_CONFIDENCE standard deviations away from our estimate. Similarly, the +// MAXIMUM_CONFIDENCE interval here means there is a one in a thousand chance +// that the true value of our estimate is outside the interval. These are used +// as bounds on the adjustments applied at the outcome of a GC cycle. +const double ShenandoahAdaptiveOldHeuristics::MINIMUM_CONFIDENCE = 0.319; // 25% +const double ShenandoahAdaptiveOldHeuristics::MAXIMUM_CONFIDENCE = 3.291; // 99.9% + +ShenandoahAdaptiveOldHeuristics::ShenandoahAdaptiveOldHeuristics(ShenandoahGeneration* generation) : + ShenandoahOldHeuristics(generation), + _margin_of_error_sd(ShenandoahAdaptiveInitialConfidence), + _spike_threshold_sd(ShenandoahAdaptiveInitialSpikeThreshold), + _last_trigger(OTHER) { } + +ShenandoahAdaptiveOldHeuristics::~ShenandoahAdaptiveOldHeuristics() {} + +void ShenandoahAdaptiveOldHeuristics::choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset, + RegionData* data, size_t size, + size_t actual_free) { + size_t garbage_threshold = ShenandoahHeapRegion::region_size_bytes() * ShenandoahGarbageThreshold / 100; + + // The logic for cset selection in adaptive is as follows: + // + // 1. We cannot get cset larger than available free space. Otherwise we guarantee OOME + // during evacuation, and thus guarantee full GC. In practice, we also want to let + // application to allocate something. This is why we limit CSet to some fraction of + // available space. In non-overloaded heap, max_cset would contain all plausible candidates + // over garbage threshold. + // + // 2. We should not get cset too low so that free threshold would not be met right + // after the cycle. Otherwise we get back-to-back cycles for no reason if heap is + // too fragmented. In non-overloaded non-fragmented heap min_garbage would be around zero. + // + // Therefore, we start by sorting the regions by garbage. Then we unconditionally add the best candidates + // before we meet min_garbage. Then we add all candidates that fit with a garbage threshold before + // we hit max_cset. When max_cset is hit, we terminate the cset selection. Note that in this scheme, + // ShenandoahGarbageThreshold is the soft threshold which would be ignored until min_garbage is hit. + + size_t capacity = _generation->soft_max_capacity(); + size_t max_cset = (size_t)((1.0 * capacity / 100 * ShenandoahEvacReserve) / ShenandoahEvacWaste); + size_t free_target = (capacity / 100 * ShenandoahMinFreeThreshold) + max_cset; + size_t min_garbage = (free_target > actual_free ? (free_target - actual_free) : 0); + + log_info(gc, ergo)("Adaptive CSet Selection. Target Free: " SIZE_FORMAT "%s, Actual Free: " + SIZE_FORMAT "%s, Max CSet: " SIZE_FORMAT "%s, Min Garbage: " SIZE_FORMAT "%s", + byte_size_in_proper_unit(free_target), proper_unit_for_byte_size(free_target), + byte_size_in_proper_unit(actual_free), proper_unit_for_byte_size(actual_free), + byte_size_in_proper_unit(max_cset), proper_unit_for_byte_size(max_cset), + byte_size_in_proper_unit(min_garbage), proper_unit_for_byte_size(min_garbage)); + + + // Better select garbage-first regions + QuickSort::sort(data, (int)size, compare_by_garbage, false); + + size_t cur_cset = 0; + size_t cur_garbage = 0; + + for (size_t idx = 0; idx < size; idx++) { + ShenandoahHeapRegion* r = data[idx]._region; + + size_t new_cset = cur_cset + r->get_live_data_bytes(); + size_t new_garbage = cur_garbage + r->garbage(); + + if (new_cset > max_cset) { + break; + } + + if ((new_garbage < min_garbage) || (r->garbage() > garbage_threshold)) { + cset->add_region(r); + cur_cset = new_cset; + cur_garbage = new_garbage; + } + } +} + +void ShenandoahAdaptiveOldHeuristics::record_cycle_start() { + ShenandoahHeuristics::record_cycle_start(); + _allocation_rate.allocation_counter_reset(); +} + +void ShenandoahAdaptiveOldHeuristics::record_success_concurrent() { + ShenandoahHeuristics::record_success_concurrent(); + + size_t available = ShenandoahHeap::heap()->free_set()->available(); + + _available.add(available); + double z_score = 0.0; + if (_available.sd() > 0) { + z_score = (available - _available.avg()) / _available.sd(); + } + + log_debug(gc, ergo)("Available: " SIZE_FORMAT " %sB, z-score=%.3f. Average available: %.1f %sB +/- %.1f %sB.", + byte_size_in_proper_unit(available), proper_unit_for_byte_size(available), + z_score, + byte_size_in_proper_unit(_available.avg()), proper_unit_for_byte_size(_available.avg()), + byte_size_in_proper_unit(_available.sd()), proper_unit_for_byte_size(_available.sd())); + + // In the case when a concurrent GC cycle completes successfully but with an + // unusually small amount of available memory we will adjust our trigger + // parameters so that they are more likely to initiate a new cycle. + // Conversely, when a GC cycle results in an above average amount of available + // memory, we will adjust the trigger parameters to be less likely to initiate + // a GC cycle. + // + // The z-score we've computed is in no way statistically related to the + // trigger parameters, but it has the nice property that worse z-scores for + // available memory indicate making larger adjustments to the trigger + // parameters. It also results in fewer adjustments as the application + // stabilizes. + // + // In order to avoid making endless and likely unnecessary adjustments to the + // trigger parameters, the change in available memory (with respect to the + // average) at the end of a cycle must be beyond these threshold values. + if (z_score < LOWEST_EXPECTED_AVAILABLE_AT_END || + z_score > HIGHEST_EXPECTED_AVAILABLE_AT_END) { + // The sign is flipped because a negative z-score indicates that the + // available memory at the end of the cycle is below average. Positive + // adjustments make the triggers more sensitive (i.e., more likely to fire). + // The z-score also gives us a measure of just how far below normal. This + // property allows us to adjust the trigger parameters proportionally. + // + // The `100` here is used to attenuate the size of our adjustments. This + // number was chosen empirically. It also means the adjustments at the end of + // a concurrent cycle are an order of magnitude smaller than the adjustments + // made for a degenerated or full GC cycle (which themselves were also + // chosen empirically). + adjust_last_trigger_parameters(z_score / -100); + } +} + +void ShenandoahAdaptiveOldHeuristics::record_success_degenerated() { + ShenandoahHeuristics::record_success_degenerated(); + // Adjust both trigger's parameters in the case of a degenerated GC because + // either of them should have triggered earlier to avoid this case. + adjust_margin_of_error(DEGENERATE_PENALTY_SD); + adjust_spike_threshold(DEGENERATE_PENALTY_SD); +} + +void ShenandoahAdaptiveOldHeuristics::record_success_full() { + ShenandoahHeuristics::record_success_full(); + // Adjust both trigger's parameters in the case of a full GC because + // either of them should have triggered earlier to avoid this case. + adjust_margin_of_error(FULL_PENALTY_SD); + adjust_spike_threshold(FULL_PENALTY_SD); +} + +static double saturate(double value, double min, double max) { + return MAX2(MIN2(value, max), min); +} + +bool ShenandoahAdaptiveOldHeuristics::should_start_gc() { + size_t max_capacity = _generation->max_capacity(); + size_t capacity = _generation->soft_max_capacity(); + size_t available = _generation->available(); + size_t allocated = _generation->bytes_allocated_since_gc_start(); + + // Make sure the code below treats available without the soft tail. + size_t soft_tail = max_capacity - capacity; + available = (available > soft_tail) ? (available - soft_tail) : 0; + + // Track allocation rate even if we decide to start a cycle for other reasons. + double rate = _allocation_rate.sample(allocated); + _last_trigger = OTHER; + + size_t min_threshold = capacity / 100 * ShenandoahMinFreeThreshold; + if (available < min_threshold) { + log_info(gc)("Trigger (%s): Free (" SIZE_FORMAT "%s) is below minimum threshold (" SIZE_FORMAT "%s)", + _generation->name(), + byte_size_in_proper_unit(available), proper_unit_for_byte_size(available), + byte_size_in_proper_unit(min_threshold), proper_unit_for_byte_size(min_threshold)); + return true; + } + + // Check if are need to learn a bit about the application + const size_t max_learn = ShenandoahLearningSteps; + if (_gc_times_learned < max_learn) { + size_t init_threshold = capacity / 100 * ShenandoahInitFreeThreshold; + if (available < init_threshold) { + log_info(gc)("Trigger (%s): Learning " SIZE_FORMAT " of " SIZE_FORMAT ". Free (" SIZE_FORMAT "%s) is below initial threshold (" SIZE_FORMAT "%s)", + _generation->name(), _gc_times_learned + 1, max_learn, + byte_size_in_proper_unit(available), proper_unit_for_byte_size(available), + byte_size_in_proper_unit(init_threshold), proper_unit_for_byte_size(init_threshold)); + return true; + } + } + + // Check if allocation headroom is still okay. This also factors in: + // 1. Some space to absorb allocation spikes + // 2. Accumulated penalties from Degenerated and Full GC + size_t allocation_headroom = available; + + size_t spike_headroom = capacity / 100 * ShenandoahAllocSpikeFactor; + size_t penalties = capacity / 100 * _gc_time_penalties; + + allocation_headroom -= MIN2(allocation_headroom, spike_headroom); + allocation_headroom -= MIN2(allocation_headroom, penalties); + + double avg_cycle_time = _gc_time_history->davg() + (_margin_of_error_sd * _gc_time_history->dsd()); + double avg_alloc_rate = _allocation_rate.upper_bound(_margin_of_error_sd); + if (avg_cycle_time > allocation_headroom / avg_alloc_rate) { + log_info(gc)("Trigger (%s): Average GC time (%.2f ms) is above the time for average allocation rate (%.0f %sB/s) to deplete free headroom (" SIZE_FORMAT "%s) (margin of error = %.2f)", + _generation->name(), avg_cycle_time * 1000, + byte_size_in_proper_unit(avg_alloc_rate), proper_unit_for_byte_size(avg_alloc_rate), + byte_size_in_proper_unit(allocation_headroom), proper_unit_for_byte_size(allocation_headroom), + _margin_of_error_sd); + + log_info(gc, ergo)("Free headroom: " SIZE_FORMAT "%s (free) - " SIZE_FORMAT "%s (spike) - " SIZE_FORMAT "%s (penalties) = " SIZE_FORMAT "%s", + byte_size_in_proper_unit(available), proper_unit_for_byte_size(available), + byte_size_in_proper_unit(spike_headroom), proper_unit_for_byte_size(spike_headroom), + byte_size_in_proper_unit(penalties), proper_unit_for_byte_size(penalties), + byte_size_in_proper_unit(allocation_headroom), proper_unit_for_byte_size(allocation_headroom)); + + _last_trigger = RATE; + return true; + } + + bool is_spiking = _allocation_rate.is_spiking(rate, _spike_threshold_sd); + if (is_spiking && avg_cycle_time > allocation_headroom / rate) { + log_info(gc)("Trigger (%s): Average GC time (%.2f ms) is above the time for instantaneous allocation rate (%.0f %sB/s) to deplete free headroom (" SIZE_FORMAT "%s) (spike threshold = %.2f)", + _generation->name(), avg_cycle_time * 1000, + byte_size_in_proper_unit(rate), proper_unit_for_byte_size(rate), + byte_size_in_proper_unit(allocation_headroom), proper_unit_for_byte_size(allocation_headroom), + _spike_threshold_sd); + _last_trigger = SPIKE; + return true; + } + + return ShenandoahHeuristics::should_start_gc(); +} + +void ShenandoahAdaptiveOldHeuristics::adjust_last_trigger_parameters(double amount) { + switch (_last_trigger) { + case RATE: + adjust_margin_of_error(amount); + break; + case SPIKE: + adjust_spike_threshold(amount); + break; + case OTHER: + // nothing to adjust here. + break; + default: + ShouldNotReachHere(); + } +} + +void ShenandoahAdaptiveOldHeuristics::adjust_margin_of_error(double amount) { + _margin_of_error_sd = saturate(_margin_of_error_sd + amount, MINIMUM_CONFIDENCE, MAXIMUM_CONFIDENCE); + log_debug(gc, ergo)("Margin of error now %.2f", _margin_of_error_sd); +} + +void ShenandoahAdaptiveOldHeuristics::adjust_spike_threshold(double amount) { + _spike_threshold_sd = saturate(_spike_threshold_sd - amount, MINIMUM_CONFIDENCE, MAXIMUM_CONFIDENCE); + log_debug(gc, ergo)("Spike threshold now: %.2f", _spike_threshold_sd); +} + diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveOldHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveOldHeuristics.hpp new file mode 100644 index 0000000000000..fce1eb8552b49 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveOldHeuristics.hpp @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHADAPTIVEOLDHEURISTICS_HPP +#define SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHADAPTIVEOLDHEURISTICS_HPP + +#include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp" +#include "gc/shenandoah/shenandoahPhaseTimings.hpp" +#include "utilities/numberSeq.hpp" + +class ShenandoahAllocationRate; + +class ShenandoahAdaptiveOldHeuristics : public ShenandoahOldHeuristics { +public: + ShenandoahAdaptiveOldHeuristics(ShenandoahGeneration* generation); + + virtual ~ShenandoahAdaptiveOldHeuristics(); + + virtual void choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset, + RegionData* data, size_t size, + size_t actual_free); + + void record_cycle_start(); + void record_success_concurrent(); + void record_success_degenerated(); + void record_success_full(); + + virtual bool should_start_gc(); + + virtual const char* name() { return "AdaptiveOld"; } + virtual bool is_diagnostic() { return false; } + virtual bool is_experimental() { return false; } + + private: + // These are used to adjust the margin of error and the spike threshold + // in response to GC cycle outcomes. These values are shared, but the + // margin of error and spike threshold trend in opposite directions. + const static double FULL_PENALTY_SD; + const static double DEGENERATE_PENALTY_SD; + + const static double MINIMUM_CONFIDENCE; + const static double MAXIMUM_CONFIDENCE; + + const static double LOWEST_EXPECTED_AVAILABLE_AT_END; + const static double HIGHEST_EXPECTED_AVAILABLE_AT_END; + + friend class ShenandoahAllocationRate; + + // Used to record the last trigger that signaled to start a GC. + // This itself is used to decide whether or not to adjust the margin of + // error for the average cycle time and allocation rate or the allocation + // spike detection threshold. + enum Trigger { + SPIKE, RATE, OTHER + }; + + void adjust_last_trigger_parameters(double amount); + void adjust_margin_of_error(double amount); + void adjust_spike_threshold(double amount); + + ShenandoahAllocationRate _allocation_rate; + + // The margin of error expressed in standard deviations to add to our + // average cycle time and allocation rate. As this value increases we + // tend to over estimate the rate at which mutators will deplete the + // heap. In other words, erring on the side of caution will trigger more + // concurrent GCs. + double _margin_of_error_sd; + + // The allocation spike threshold is expressed in standard deviations. + // If the standard deviation of the most recent sample of the allocation + // rate exceeds this threshold, a GC cycle is started. As this value + // decreases the sensitivity to allocation spikes increases. In other + // words, lowering the spike threshold will tend to increase the number + // of concurrent GCs. + double _spike_threshold_sd; + + // Remember which trigger is responsible for the last GC cycle. When the + // outcome of the cycle is evaluated we will adjust the parameters for the + // corresponding triggers. Note that successful outcomes will raise + // the spike threshold and lower the margin of error. + Trigger _last_trigger; + + // Keep track of the available memory at the end of a GC cycle. This + // establishes what is 'normal' for the application and is used as a + // source of feedback to adjust trigger parameters. + TruncatedSeq _available; +}; + +#endif // SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHADAPTIVEOLDHEURISTICS_HPP diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveOldHeuristics.cpp new file mode 100644 index 0000000000000..2ba15a7d410b8 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveOldHeuristics.cpp @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "gc/shenandoah/heuristics/shenandoahAggressiveOldHeuristics.hpp" +#include "gc/shenandoah/shenandoahCollectionSet.hpp" +#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" +#include "logging/log.hpp" +#include "logging/logTag.hpp" +#include "runtime/os.hpp" + +ShenandoahAggressiveOldHeuristics::ShenandoahAggressiveOldHeuristics(ShenandoahGeneration* generation) : ShenandoahOldHeuristics(generation) { + // Do not shortcut evacuation + SHENANDOAH_ERGO_OVERRIDE_DEFAULT(ShenandoahImmediateThreshold, 100); + + // Aggressive evacuates everything, so it needs as much evac space as it can get + SHENANDOAH_ERGO_ENABLE_FLAG(ShenandoahEvacReserveOverflow); + + // If class unloading is globally enabled, aggressive does unloading even with + // concurrent cycles. + if (ClassUnloading) { + SHENANDOAH_ERGO_OVERRIDE_DEFAULT(ShenandoahUnloadClassesFrequency, 1); + } +} + +void ShenandoahAggressiveOldHeuristics::choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset, + RegionData* data, size_t size, + size_t free) { + for (size_t idx = 0; idx < size; idx++) { + ShenandoahHeapRegion* r = data[idx]._region; + if (r->garbage() > 0) { + cset->add_region(r); + } + } +} + +bool ShenandoahAggressiveOldHeuristics::should_start_gc() { + log_info(gc)("Trigger: Start next cycle immediately"); + return true; +} + +bool ShenandoahAggressiveOldHeuristics::should_unload_classes() { + if (!can_unload_classes_normal()) return false; + if (has_metaspace_oom()) return true; + // Randomly unload classes with 50% chance. + return (os::random() & 1) == 1; +} diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveOldHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveOldHeuristics.hpp new file mode 100644 index 0000000000000..8f7b699ba9cfa --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveOldHeuristics.hpp @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHAGGRESSIVEOLDHEURISTICS_HPP +#define SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHAGGRESSIVEOLDHEURISTICS_HPP + +#include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp" + +class ShenandoahAggressiveOldHeuristics : public ShenandoahOldHeuristics { +public: + ShenandoahAggressiveOldHeuristics(ShenandoahGeneration* generation); + + virtual void choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset, + RegionData* data, size_t size, + size_t free); + + virtual bool should_start_gc(); + + virtual bool should_unload_classes(); + + virtual const char* name() { return "AggressiveOld"; } + virtual bool is_diagnostic() { return true; } + virtual bool is_experimental() { return false; } +}; + +#endif // SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHAGGRESSIVEOLDHEURISTICS_HPP diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactOldHeuristics.cpp new file mode 100644 index 0000000000000..3fb24256d1b29 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactOldHeuristics.cpp @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "gc/shenandoah/shenandoahCollectionSet.hpp" +#include "gc/shenandoah/heuristics/shenandoahCompactOldHeuristics.hpp" +#include "gc/shenandoah/shenandoahFreeSet.hpp" +#include "gc/shenandoah/shenandoahGeneration.hpp" +#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" +#include "logging/log.hpp" +#include "logging/logTag.hpp" + +ShenandoahCompactOldHeuristics::ShenandoahCompactOldHeuristics(ShenandoahGeneration* generation) : + ShenandoahOldHeuristics(generation) { + SHENANDOAH_ERGO_ENABLE_FLAG(ExplicitGCInvokesConcurrent); + SHENANDOAH_ERGO_ENABLE_FLAG(ShenandoahImplicitGCInvokesConcurrent); + SHENANDOAH_ERGO_ENABLE_FLAG(ShenandoahUncommit); + SHENANDOAH_ERGO_ENABLE_FLAG(ShenandoahAlwaysClearSoftRefs); + SHENANDOAH_ERGO_OVERRIDE_DEFAULT(ShenandoahAllocationThreshold, 10); + SHENANDOAH_ERGO_OVERRIDE_DEFAULT(ShenandoahImmediateThreshold, 100); + SHENANDOAH_ERGO_OVERRIDE_DEFAULT(ShenandoahUncommitDelay, 1000); + SHENANDOAH_ERGO_OVERRIDE_DEFAULT(ShenandoahGuaranteedGCInterval, 30000); + SHENANDOAH_ERGO_OVERRIDE_DEFAULT(ShenandoahGarbageThreshold, 10); +} + +bool ShenandoahCompactOldHeuristics::should_start_gc() { + size_t max_capacity = _generation->max_capacity(); + size_t capacity = _generation->soft_max_capacity(); + size_t available = _generation->available(); + + // Make sure the code below treats available without the soft tail. + size_t soft_tail = max_capacity - capacity; + available = (available > soft_tail) ? (available - soft_tail) : 0; + + size_t threshold_bytes_allocated = capacity / 100 * ShenandoahAllocationThreshold; + size_t min_threshold = capacity / 100 * ShenandoahMinFreeThreshold; + + if (available < min_threshold) { + log_info(gc)("Trigger: Free (" SIZE_FORMAT "%s) is below minimum threshold (" SIZE_FORMAT "%s)", + byte_size_in_proper_unit(available), proper_unit_for_byte_size(available), + byte_size_in_proper_unit(min_threshold), proper_unit_for_byte_size(min_threshold)); + return true; + } + + size_t bytes_allocated = _generation->bytes_allocated_since_gc_start(); + if (bytes_allocated > threshold_bytes_allocated) { + log_info(gc)("Trigger: Allocated since last cycle (" SIZE_FORMAT "%s) is larger than allocation threshold (" SIZE_FORMAT "%s)", + byte_size_in_proper_unit(bytes_allocated), proper_unit_for_byte_size(bytes_allocated), + byte_size_in_proper_unit(threshold_bytes_allocated), proper_unit_for_byte_size(threshold_bytes_allocated)); + return true; + } + + return ShenandoahHeuristics::should_start_gc(); +} + +void ShenandoahCompactOldHeuristics::choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset, + RegionData* data, size_t size, + size_t actual_free) { + // Do not select too large CSet that would overflow the available free space + size_t max_cset = actual_free * 3 / 4; + + log_info(gc, ergo)("CSet Selection. Actual Free: " SIZE_FORMAT "%s, Max CSet: " SIZE_FORMAT "%s", + byte_size_in_proper_unit(actual_free), proper_unit_for_byte_size(actual_free), + byte_size_in_proper_unit(max_cset), proper_unit_for_byte_size(max_cset)); + + size_t threshold = ShenandoahHeapRegion::region_size_bytes() * ShenandoahGarbageThreshold / 100; + + size_t live_cset = 0; + for (size_t idx = 0; idx < size; idx++) { + ShenandoahHeapRegion* r = data[idx]._region; + size_t new_cset = live_cset + r->get_live_data_bytes(); + if (new_cset < max_cset && r->garbage() > threshold) { + live_cset = new_cset; + cset->add_region(r); + } + } +} diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactOldHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactOldHeuristics.hpp new file mode 100644 index 0000000000000..3297744ce8910 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactOldHeuristics.hpp @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHCOMPACTOLDHEURISTICS_HPP +#define SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHCOMPACTOLDHEURISTICS_HPP + +#include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp" + +class ShenandoahCompactOldHeuristics : public ShenandoahOldHeuristics { +public: + ShenandoahCompactOldHeuristics(ShenandoahGeneration* generation); + + virtual bool should_start_gc(); + + virtual void choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset, + RegionData* data, size_t size, + size_t actual_free); + + virtual const char* name() { return "CompactOld"; } + virtual bool is_diagnostic() { return false; } + virtual bool is_experimental() { return false; } +}; + +#endif // SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHCOMPACTOLDHEURISTICS_HPP diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp index 43a35bf1361f9..ae21f753df2ba 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018, 2020, Red Hat, Inc. All rights reserved. + * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -46,6 +46,7 @@ int ShenandoahHeuristics::compare_by_garbage(RegionData a, RegionData b) { } ShenandoahHeuristics::ShenandoahHeuristics(ShenandoahGeneration* generation) : + _generation(generation), _region_data(NULL), _degenerated_cycles_in_a_row(0), _successful_cycles_in_a_row(0), @@ -54,8 +55,7 @@ ShenandoahHeuristics::ShenandoahHeuristics(ShenandoahGeneration* generation) : _gc_times_learned(0), _gc_time_penalties(0), _gc_time_history(new TruncatedSeq(10, ShenandoahAdaptiveDecayFactor)), - _metaspace_oom(), - _generation(generation) + _metaspace_oom() { // No unloading during concurrent mark? Communicate that to heuristics if (!ClassUnloadingWithConcurrentMark) { @@ -72,11 +72,12 @@ ShenandoahHeuristics::~ShenandoahHeuristics() { FREE_C_HEAP_ARRAY(RegionGarbage, _region_data); } -void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collection_set) { - assert(collection_set->count() == 0, "Must be empty"); - +void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collection_set, ShenandoahOldHeuristics* old_heuristics) { ShenandoahHeap* heap = ShenandoahHeap::heap(); + assert(collection_set->count() == 0, "Must be empty"); + assert(_generation->generation_mode() != OLD, "Old GC invokes ShenandoahOldHeuristics::choose_collection_set()"); + // Check all pinned regions have updated status before choosing the collection set. heap->assert_pinned_region_status(); @@ -116,11 +117,8 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec immediate_regions++; immediate_garbage += garbage; region->make_trash_immediate(); - } else if (_generation->generation_mode() != OLD) { - // HEY! At this stage in development our concurrent old - // marking does NOT complete the subsequent phases of the collection - // and we don't want regions stuck in the 'in_cset' state because - // various asserts will trip. + } else { + assert (_generation->generation_mode() != OLD, "OLD is handled elsewhere"); // This is our candidate for later consideration. candidates[cand_idx]._region = region; @@ -162,6 +160,14 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec size_t immediate_percent = (total_garbage == 0) ? 0 : (immediate_garbage * 100 / total_garbage); if (immediate_percent <= ShenandoahImmediateThreshold) { + + if (old_heuristics != NULL) { + old_heuristics->prime_collection_set(collection_set); + } + // else, this is global collection and doesn't need to prime_collection_set + + // Add young-gen regions into the collection set. This is a virtual call, implemented differently by each + // of the heuristics subclasses. choose_collection_set_from_regiondata(collection_set, candidates, cand_idx, immediate_garbage + free); } @@ -221,7 +227,7 @@ bool ShenandoahHeuristics::should_degenerate_cycle() { void ShenandoahHeuristics::adjust_penalty(intx step) { assert(0 <= _gc_time_penalties && _gc_time_penalties <= 100, - "In range before adjustment: " INTX_FORMAT, _gc_time_penalties); + "In range before adjustment: " INTX_FORMAT, _gc_time_penalties); intx new_val = _gc_time_penalties + step; if (new_val < 0) { @@ -233,7 +239,7 @@ void ShenandoahHeuristics::adjust_penalty(intx step) { _gc_time_penalties = new_val; assert(0 <= _gc_time_penalties && _gc_time_penalties <= 100, - "In range after adjustment: " INTX_FORMAT, _gc_time_penalties); + "In range after adjustment: " INTX_FORMAT, _gc_time_penalties); } void ShenandoahHeuristics::record_success_concurrent() { @@ -303,7 +309,8 @@ double ShenandoahHeuristics::time_since_last_gc() const { } bool ShenandoahHeuristics::in_generation(ShenandoahHeapRegion* region) { - return (_generation->generation_mode() == GLOBAL) - || (_generation->generation_mode() == YOUNG && region->affiliation() == YOUNG_GENERATION) - || (_generation->generation_mode() == OLD && region->affiliation() == OLD_GENERATION); + return ((_generation->generation_mode() == GLOBAL) + || (_generation->generation_mode() == YOUNG && region->affiliation() == YOUNG_GENERATION) + || (_generation->generation_mode() == OLD && region->affiliation() == OLD_GENERATION)); } + diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp index 52a2bcf3f2bc9..1c9e1c1f07ac0 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp @@ -69,6 +69,21 @@ class ShenandoahHeuristics : public CHeapObj { size_t _garbage; } RegionData; + ShenandoahGeneration* _generation; + + // if (_generation->generation_mode() == GLOBAL) _region_data represents + // the results of most recently completed global marking pass + // if (_generation->generation_mode() == OLD) _region_data represents + // the results of most recently completed old-gen marking pass + // if (_generation->generation_mode() == YOUNG) _region_data represents + // the resulits of most recently completed young-gen marking pass + // + // Note that there is some redundancy represented in _region_data because + // each instance is an array large enough to hold all regions. However, + // any region in young-gen is not in old-gen. And any time we are + // making use of the GLOBAL data, there is no need to maintain the + // YOUNG or OLD data. Consider this redundancy of data structure to + // have negligible cost unless proven otherwise. RegionData* _region_data; uint _degenerated_cycles_in_a_row; @@ -84,10 +99,12 @@ class ShenandoahHeuristics : public CHeapObj { // There may be many threads that contend to set this flag ShenandoahSharedFlag _metaspace_oom; - ShenandoahGeneration* _generation; - static int compare_by_garbage(RegionData a, RegionData b); + // TODO: We need to enhance this API to give visibility to accompanying old-gen evacuation effort. + // In the case that the old-gen evacuation effort is small or zero, the young-gen heuristics + // should feel free to dedicate increased efforts to young-gen evacuation. + virtual void choose_collection_set_from_regiondata(ShenandoahCollectionSet* set, RegionData* data, size_t data_size, size_t free) = 0; @@ -122,7 +139,7 @@ class ShenandoahHeuristics : public CHeapObj { virtual void record_requested_gc(); - virtual void choose_collection_set(ShenandoahCollectionSet* collection_set); + virtual void choose_collection_set(ShenandoahCollectionSet* collection_set, ShenandoahOldHeuristics* old_heuristics); virtual bool can_unload_classes(); virtual bool can_unload_classes_normal(); diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp new file mode 100644 index 0000000000000..e6ba41679d925 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp @@ -0,0 +1,317 @@ +/* + * Copyright (c) 2021, Amazon.com, Inc. or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp" +#include "utilities/quickSort.hpp" + +ShenandoahOldHeuristics::ShenandoahOldHeuristics(ShenandoahGeneration* generation) : + ShenandoahHeuristics(generation), + _old_collection_candidates(0), + _next_old_collection_candidate(0), + _hidden_old_collection_candidates(0), + _hidden_next_old_collection_candidate(0), + _old_coalesce_and_fill_candidates(0), + _first_coalesce_and_fill_candidate(0) +{ +} + +void ShenandoahOldHeuristics::prime_collection_set(ShenandoahCollectionSet* collection_set) { + uint included_old_regions = 0; + size_t evacuated_old_bytes = 0; + + // TODO: + // The max_old_evacuation_bytes and promotion_budget_bytes constants represent a first + // approximation to desired operating parameters. Eventually, these values should be determined + // by heuristics and should adjust dynamically based on most current execution behavior. In the + // interrim, we may choose to offer command-line options to set the values of these configuration + // parameters. + + // max_old_evacuation_bytes represents an "arbitrary" bound on how much evacuation effort is dedicated + // to old-gen regions. + const size_t max_old_evacuation_bytes = (ShenandoahHeapRegion::region_size_bytes() * 8); + + // promotion_budget_bytes represents an "arbitrary" bound on how many bytes can be consumed by young-gen + // objects promoted into old-gen memory. We need to avoid a scenario under which promotion of objects + // depletes old-gen available memory to the point that there is insufficient memory to hold old-gen objects + // that need to be evacuated from within the old-gen collection set. + // + // TODO We should probably enforce this, but there is no enforcement currently. Key idea: if there is not + // sufficient memory within old-gen to hold an object that wants to be promoted, defer promotion until a + // subsequent evacuation pass. Since enforcement may be expensive, requiring frequent synchronization + // between mutator and GC threads, here's an alternative "greedy" mitigation strategy: Set the parameter's + // value so overflow is "very rare". In the case that we experience overflow, evacuate what we can from + // within the old collection set, but don't evacuate everything. At the end of evacuation, any collection + // set region that was not fully evacuated cannot be recycled. It becomes a prime candidate for the next + // collection set selection. Here, we'd rather fall back to this contingent behavior than force a full STW + // collection. + const size_t promotion_budget_bytes = (ShenandoahHeapRegion::region_size_bytes() / 2); + + // old_evacuation_budget is an upper bound on the amount of live memory that can be evacuated. + // + // If a region is put into the collection set, then this region's free (not yet used) bytes are no longer + // "available" to hold the results of other evacuations. This may cause a decrease in the remaining amount + // of memory that can still be evacuated. We address this by reducing the evacuation budget by the amount + // of live memory in that region and by the amount of unallocated memory in that region if the evacuation + // budget is constrained by availability of free memory. See remaining_old_evacuation_budget below. + + // Allow no more evacuation than exists free-space within old-gen memory + size_t old_evacuation_budget = ((_generation->available() > promotion_budget_bytes) + ? _generation->available() - promotion_budget_bytes: 0); + + // But if the amount of available free space in old-gen memory exceeds the pacing bound on how much old-gen + // memory can be evacuated during each evacuation pass, then cut the old-gen evacuation further. The pacing + // bound is designed to assure that old-gen evacuations to not excessively slow the evacuation pass in order + // to assure that young-gen GC cadence is not disrupted. + + // excess_free_capacity represents availability of memory to hold evacuations beyond what is required to hold + // planned evacuations. It may go negative if we choose to collect regions with large amounts of free memory. + long long excess_free_capacity; + if (old_evacuation_budget > max_old_evacuation_bytes) { + excess_free_capacity = old_evacuation_budget - max_old_evacuation_bytes; + old_evacuation_budget = max_old_evacuation_bytes; + } else + excess_free_capacity = 0; + + size_t remaining_old_evacuation_budget = old_evacuation_budget; + + // The number of old-gen regions that were selected as candidates for collection at the end of the most recent + // old-gen concurrent marking phase and have not yet been collected is represented by + // unprocessed_old_collection_candidates() + while (unprocessed_old_collection_candidates() > 0) { + // Old collection candidates are sorted in order of decreasing garbage contained therein. + ShenandoahHeapRegion* r = next_old_collection_candidate(); + + // Assuming region r is added to the collection set, what will be the remaining_old_evacuation_budget after + // accounting for the loss of region r's free() memory. + size_t adjusted_remaining_old_evacuation_budget; + + // If we choose region r to be collected, then we need to decrease the capacity to hold other evacuations by + // the size of r's free memory. + excess_free_capacity -= r->free(); + // If subtracting r->free from excess_free_capacity() makes it go negative, that means we are going to have + // to decrease the evacuation budget. + if (excess_free_capacity < 0) { + if (remaining_old_evacuation_budget < (size_t) -excess_free_capacity) { + // By setting adjusted_remaining_old_evacuation_budget to 0, we prevent further additions to the old-gen + // collection set, unless the region has zero live data bytes. + adjusted_remaining_old_evacuation_budget = 0; + } else { + // Adding negative excess_free_capacity decreases the adjusted_remaining_old_evacuation_budget + adjusted_remaining_old_evacuation_budget = remaining_old_evacuation_budget + excess_free_capacity; + } + } else { + adjusted_remaining_old_evacuation_budget = remaining_old_evacuation_budget; + } + + if (r->get_live_data_bytes() > adjusted_remaining_old_evacuation_budget) { + break; + } + collection_set->add_region(r); + included_old_regions++; + evacuated_old_bytes += r->get_live_data_bytes(); + consume_old_collection_candidate(); + remaining_old_evacuation_budget = adjusted_remaining_old_evacuation_budget - r->get_live_data_bytes(); + } + + if (included_old_regions > 0) { + log_info(gc)("Old-gen piggyback evac (%llu regions, %llu bytes)", + (unsigned long long) included_old_regions, + (unsigned long long) evacuated_old_bytes); + } +} + + +void ShenandoahOldHeuristics::choose_collection_set(ShenandoahCollectionSet* collection_set, ShenandoahOldHeuristics* old_heuristics) { + assert(collection_set->count() == 0, "Must be empty"); + + // Old-gen doesn't actually choose a collection set to be evacuated by its own gang of worker tasks. + // Instead, it computes the set of regions to be evacuated by subsequent young-gen evacuation passes. + prepare_for_old_collections(); +} + +void ShenandoahOldHeuristics::prepare_for_old_collections() { + assert(_generation->generation_mode() == OLD, "This service only available for old-gc heuristics"); + ShenandoahHeap* heap = ShenandoahHeap::heap(); + uint free_regions = 0; + size_t cand_idx = 0; + size_t total_garbage = 0; + size_t num_regions = heap->num_regions(); + + RegionData* candidates = _region_data; + for (size_t i = 0; i < num_regions; i++) { + ShenandoahHeapRegion* region = heap->get_region(i); + if (!in_generation(region)) { + continue; + } else { + size_t garbage = region->garbage(); + total_garbage += garbage; + + candidates[cand_idx]._region = region; + candidates[cand_idx]._garbage = garbage; + cand_idx++; + } + } + + // Give special treatment to humongous regions. Assume humongous regions is entirely + // garbage or entirely non-garbage. Assume that a head humongous region and the associated + // humongous continuous regions are uniformly entirely garbage or entirely non-garbage. + // + // Sift garbage humongous regions to front, non-garbage humongous regions to end of array. + size_t first_non_humongous_empty = 0; + size_t first_humongous_non_empty = cand_idx; + + size_t i = 0; + while (i < first_humongous_non_empty) { + ShenandoahHeapRegion* region = candidates[i]._region; + if (region->is_humongous()) { + if (region->get_live_data_bytes() == 0) { + // Humongous region is entirely garbage. Reclaim it. + if (i == first_non_humongous_empty) { + first_non_humongous_empty++; + } else { + RegionData swap_tmp = candidates[i]; + candidates[i] = candidates[first_non_humongous_empty]; + candidates[first_non_humongous_empty++] = swap_tmp; + } + i++; + } else { + // Humongous region is non garbage. Don't reclaim it. + if (i + 1 == first_humongous_non_empty) { + first_humongous_non_empty--; + i++; + } else { + RegionData swap_tmp = candidates[i]; + candidates[i] = candidates[--first_humongous_non_empty]; + candidates[first_humongous_non_empty] = swap_tmp; + // Do not increment i so we can revisit swapped entry on next iteration + } + } + } else { + i++; + } + } + + + // Prioritize regions to select garbage-first regions + QuickSort::sort(candidates + first_non_humongous_empty, (int)(first_humongous_non_empty - first_non_humongous_empty), + compare_by_garbage, false); + + // Any old-gen region that contains (ShenandoahGarbageThreshold (default value 25))% garbage or more is to + // be evacuated. + // + // TODO: it probably makes sense to define old-generation collection_threshold_garbage_percent differently + // than the young-gen ShenandoahGarbageThreshold. So a different command-line option might allow specification + // distinct values for each. Even better, allow collection_threshold_garbage_percent to be determined + // adaptively, by heuristics. + + const size_t collection_threshold_garbage_percent = ShenandoahGarbageThreshold; + + size_t region_size = ShenandoahHeapRegion::region_size_bytes(); + for (size_t i = first_non_humongous_empty; i < first_humongous_non_empty; i++) { + // Do approximate percent to avoid floating point math + size_t percent_garbage = candidates[i]._garbage * 100 / region_size; + + if (percent_garbage < collection_threshold_garbage_percent) { + _hidden_next_old_collection_candidate = 0; + _hidden_old_collection_candidates = i; + _first_coalesce_and_fill_candidate = i; + _old_coalesce_and_fill_candidates = cand_idx - i; + + // Note that we do not coalesce and fill occupied humongous regions + // HR: humongous regions, RR: regular regions, CF: coalesce and fill regions + log_info(gc)("Old-gen mark evac (%u HR, %llu RR), %llu CF)", + (unsigned int) first_non_humongous_empty, + (unsigned long long) (_hidden_old_collection_candidates - first_non_humongous_empty), + (unsigned long long) _old_coalesce_and_fill_candidates); + return; + } + } + + // If we reach here, all of non-humogous old-gen regions are candidates for collection set. + _hidden_next_old_collection_candidate = 0; + _hidden_old_collection_candidates = first_humongous_non_empty; + _first_coalesce_and_fill_candidate = 0; + _old_coalesce_and_fill_candidates = 0; + + // Note that we do not coalesce and fill occupied humongous regions + // HR: humongous regions, RR: regular regions, CF: coalesce and fill regions + log_info(gc)("Old-gen mark evac (%u HR, %llu RR), %llu CF)", + (unsigned int) first_non_humongous_empty, + (unsigned long long) (_hidden_old_collection_candidates - first_non_humongous_empty), + (unsigned long long) _old_coalesce_and_fill_candidates); +} + +void ShenandoahOldHeuristics::start_old_evacuations() { + assert(_generation->generation_mode() == OLD, "This service only available for old-gc heuristics"); + + _old_collection_candidates = _hidden_old_collection_candidates; + _next_old_collection_candidate = _hidden_next_old_collection_candidate; + + _hidden_old_collection_candidates = 0;} + + +uint ShenandoahOldHeuristics::unprocessed_old_collection_candidates() { + assert(_generation->generation_mode() == OLD, "This service only available for old-gc heuristics"); + return _old_collection_candidates + _hidden_old_collection_candidates; +} + +ShenandoahHeapRegion* ShenandoahOldHeuristics::next_old_collection_candidate() { + assert(_generation->generation_mode() == OLD, "This service only available for old-gc heuristics"); + return _region_data[_next_old_collection_candidate]._region; +} + +void ShenandoahOldHeuristics::consume_old_collection_candidate() { + assert(_generation->generation_mode() == OLD, "This service only available for old-gc heuristics"); + _next_old_collection_candidate++; + _old_collection_candidates--; +} + +uint ShenandoahOldHeuristics::old_coalesce_and_fill_candidates() { + assert(_generation->generation_mode() == OLD, "This service only available for old-gc heuristics"); + return _old_coalesce_and_fill_candidates; +} + +void ShenandoahOldHeuristics::get_coalesce_and_fill_candidates(ShenandoahHeapRegion** buffer) { + assert(_generation->generation_mode() == OLD, "This service only available for old-gc heuristics"); + uint count = _old_coalesce_and_fill_candidates; + int index = _first_coalesce_and_fill_candidate; + while (count-- > 0) { + *buffer++ = _region_data[index++]._region; + } +} + +bool ShenandoahOldHeuristics::should_defer_gc() { + if (unprocessed_old_collection_candidates() > 0) { + // Cannot start a new old-gen GC until previous one has finished. + // + // Future refinement: under certain circumstances, we might be more sophisticated about this choice. + // For example, we could choose to abandon the prevoius old collection before it has completed evacuations, + // but this would require that we coalesce and fill all garbage within unevacuated collection-set regions. + return true; + } + return false; +} + diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp new file mode 100644 index 0000000000000..f78fe89ff0f83 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2021, Amazon.com, Inc. or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHOLDHEURISTICS_HPP +#define SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHOLDHEURISTICS_HPP + +#include "gc/shenandoah/shenandoahCollectionSet.inline.hpp" +#include "gc/shenandoah/shenandoahGeneration.hpp" +#include "gc/shenandoah/shenandoahHeap.inline.hpp" +#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" +#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" + +class ShenandoahOldHeuristics : public ShenandoahHeuristics { + +protected: + + // if (_generation->generation_mode() == OLD) _old_collection_candidates + // represent the number of regions selected for collection following the + // most recently completed old-gen mark that have not yet been selected + // for evacuation and _next_collection_candidate is the index within + // _region_data of the next candidate region to be selected for evacuation. + // if (_generation->generation_mode() != OLD) these two variables are + // not used. + uint _old_collection_candidates; + uint _next_old_collection_candidate; + + // At the time we select the old-gen collection set, _hidden_old_collection_candidates + // and _hidden_next_old_collection_candidates are set to remember the intended old-gen + // collection set. After all old-gen regions not in the old-gen collection set have been + // coalesced and filled, the content of these variables is copied to _old_collection_candidates + // and _next_old_collection_candidates so that evacuations can begin evacuating these regions. + uint _hidden_old_collection_candidates; + uint _hidden_next_old_collection_candidate; + + // if (_generation->generation_mode() == OLD) + // _old_coalesce_and_fill_candidates represents the number of regions + // that were chosen for the garbage contained therein to be coalesced + // and filled and _first_coalesce_and_fill_candidate represents the + // the index of the first such region within the _region_data array. + // if (_generation->generation_mode() != OLD) these two variables are + // not used. + uint _old_coalesce_and_fill_candidates; + uint _first_coalesce_and_fill_candidate; + + // Prepare for evacuation of old-gen regions by capturing the mark results of a recently completed concurrent mark pass. + void prepare_for_old_collections(); + +public: + ShenandoahOldHeuristics(ShenandoahGeneration* generation); + + virtual void choose_collection_set(ShenandoahCollectionSet* collection_set, ShenandoahOldHeuristics* old_heuristics); + + void prime_collection_set(ShenandoahCollectionSet* set); + + // Having coalesced and filled all old-gen heap regions that are not part of the old-gen collection set, begin + // evacuating the collection set. + void start_old_evacuations(); + + // How many old-collection candidates have not yet been processed? + uint unprocessed_old_collection_candidates(); + + // Return the next old-collection candidate in order of decreasing amounts of garbage. (We process most-garbage regions + // first.) This does not consume the candidate. If the candidate is selected for inclusion in a collection set, then + // the candidate is consumed by invoking consume_old_collection_candidate(). + ShenandoahHeapRegion* next_old_collection_candidate(); + + // Adjust internal state to reflect that one fewer old-collection candidate remains to be processed. + void consume_old_collection_candidate(); + + // How many old-collection regions were identified at the end of the most recent old-gen mark to require their + // unmarked objects to be coalesced and filled? + uint old_coalesce_and_fill_candidates(); + + // Fill in buffer with all of the old-collection regions that were identified at the end of the most recent old-gen + // mark to require their unmarked objects to be coalesced and filled. The buffer array must have at least + // old_coalesce_and_fill_candidates() entries, or memory may be corrupted when this function overwrites the + // end of the array. + void get_coalesce_and_fill_candidates(ShenandoahHeapRegion** buffer); + + bool should_defer_gc(); + +}; + +#endif // SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHOLDHEURISTICS_HPP diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveOldHeuristics.cpp new file mode 100644 index 0000000000000..eba854ae760d1 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveOldHeuristics.cpp @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "gc/shenandoah/heuristics/shenandoahPassiveOldHeuristics.hpp" +#include "gc/shenandoah/shenandoahCollectionSet.hpp" +#include "gc/shenandoah/shenandoahHeap.inline.hpp" +#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" +#include "logging/log.hpp" +#include "logging/logTag.hpp" + +bool ShenandoahPassiveOldHeuristics::should_start_gc() { + // Never do concurrent GCs. + return false; +} + +bool ShenandoahPassiveOldHeuristics::should_unload_classes() { + // Always unload classes, if we can. + return can_unload_classes(); +} + +bool ShenandoahPassiveOldHeuristics::should_degenerate_cycle() { + // Always fail to Degenerated GC, if enabled + return ShenandoahDegeneratedGC; +} + +void ShenandoahPassiveOldHeuristics::choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset, + RegionData* data, size_t size, + size_t actual_free) { + assert(ShenandoahDegeneratedGC, "This path is only taken for Degenerated GC"); + + // Do not select too large CSet that would overflow the available free space. + // Take at least the entire evacuation reserve, and be free to overflow to free space. + size_t max_capacity = ShenandoahHeap::heap()->max_capacity(); + size_t available = MAX2(max_capacity / 100 * ShenandoahEvacReserve, actual_free); + size_t max_cset = (size_t)(available / ShenandoahEvacWaste); + + log_info(gc, ergo)("CSet Selection. Actual Free: " SIZE_FORMAT "%s, Max CSet: " SIZE_FORMAT "%s", + byte_size_in_proper_unit(actual_free), proper_unit_for_byte_size(actual_free), + byte_size_in_proper_unit(max_cset), proper_unit_for_byte_size(max_cset)); + + size_t threshold = ShenandoahHeapRegion::region_size_bytes() * ShenandoahGarbageThreshold / 100; + + size_t live_cset = 0; + for (size_t idx = 0; idx < size; idx++) { + ShenandoahHeapRegion* r = data[idx]._region; + size_t new_cset = live_cset + r->get_live_data_bytes(); + if (new_cset < max_cset && r->garbage() > threshold) { + live_cset = new_cset; + cset->add_region(r); + } + } +} diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveOldHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveOldHeuristics.hpp new file mode 100644 index 0000000000000..798a908fac313 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveOldHeuristics.hpp @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHPASSIVEOLDHEURISTICS_HPP +#define SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHPASSIVEOLDHEURISTICS_HPP + +#include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp" + +class ShenandoahPassiveOldHeuristics : public ShenandoahOldHeuristics { +public: + ShenandoahPassiveOldHeuristics(ShenandoahGeneration* generation) + : ShenandoahOldHeuristics(generation) {} + + virtual bool should_start_gc(); + + virtual bool should_unload_classes(); + + virtual bool should_degenerate_cycle(); + + virtual void choose_collection_set_from_regiondata(ShenandoahCollectionSet* set, + RegionData* data, size_t data_size, + size_t free); + + virtual const char* name() { return "PassiveOld"; } + virtual bool is_diagnostic() { return true; } + virtual bool is_experimental() { return false; } +}; + +#endif // SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHPASSIVEOLDHEURISTICS_HPP diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticOldHeuristics.cpp new file mode 100644 index 0000000000000..2f6b00a4c91b6 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticOldHeuristics.cpp @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" + +#include "gc/shenandoah/heuristics/shenandoahStaticOldHeuristics.hpp" +#include "gc/shenandoah/shenandoahCollectionSet.hpp" +#include "gc/shenandoah/shenandoahFreeSet.hpp" +#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" +#include "gc/shenandoah/shenandoahGeneration.hpp" +#include "logging/log.hpp" +#include "logging/logTag.hpp" + +ShenandoahStaticOldHeuristics::ShenandoahStaticOldHeuristics(ShenandoahGeneration* generation) : + ShenandoahOldHeuristics(generation) { + SHENANDOAH_ERGO_ENABLE_FLAG(ExplicitGCInvokesConcurrent); + SHENANDOAH_ERGO_ENABLE_FLAG(ShenandoahImplicitGCInvokesConcurrent); +} + +ShenandoahStaticOldHeuristics::~ShenandoahStaticOldHeuristics() {} + +bool ShenandoahStaticOldHeuristics::should_start_gc() { + size_t max_capacity = _generation->max_capacity(); + size_t capacity = _generation->soft_max_capacity(); + size_t available = _generation->available(); + + // Make sure the code below treats available without the soft tail. + size_t soft_tail = max_capacity - capacity; + available = (available > soft_tail) ? (available - soft_tail) : 0; + + size_t threshold_available = capacity / 100 * ShenandoahMinFreeThreshold; + + if (available < threshold_available) { + log_info(gc)("Trigger: Free (" SIZE_FORMAT "%s) is below minimum threshold (" SIZE_FORMAT "%s)", + byte_size_in_proper_unit(available), proper_unit_for_byte_size(available), + byte_size_in_proper_unit(threshold_available), proper_unit_for_byte_size(threshold_available)); + return true; + } + return ShenandoahHeuristics::should_start_gc(); +} + +void ShenandoahStaticOldHeuristics::choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset, + RegionData* data, size_t size, + size_t free) { + size_t threshold = ShenandoahHeapRegion::region_size_bytes() * ShenandoahGarbageThreshold / 100; + + for (size_t idx = 0; idx < size; idx++) { + ShenandoahHeapRegion* r = data[idx]._region; + if (r->garbage() > threshold) { + cset->add_region(r); + } + } +} diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticOldHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticOldHeuristics.hpp new file mode 100644 index 0000000000000..b3aa3a6b90aed --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticOldHeuristics.hpp @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHSTATICOLDHEURISTICS_HPP +#define SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHSTATICOLDHEURISTICS_HPP + +#include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp" + +class ShenandoahStaticOldHeuristics : public ShenandoahOldHeuristics { +public: + ShenandoahStaticOldHeuristics(ShenandoahGeneration* generation); + + virtual ~ShenandoahStaticOldHeuristics(); + + virtual bool should_start_gc(); + + virtual void choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset, + RegionData* data, size_t size, + size_t free); + + virtual const char* name() { return "StaticOld"; } + virtual bool is_diagnostic() { return false; } + virtual bool is_experimental() { return false; } +}; + +#endif // SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHSTATICOLDHEURISTICS_HPP diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.cpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.cpp index 7ce4b4d0da5d7..6011bca8622b4 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.cpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.cpp @@ -24,9 +24,13 @@ #include "precompiled.hpp" #include "gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp" +#include "gc/shenandoah/heuristics/shenandoahAdaptiveOldHeuristics.hpp" #include "gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.hpp" +#include "gc/shenandoah/heuristics/shenandoahAggressiveOldHeuristics.hpp" #include "gc/shenandoah/heuristics/shenandoahCompactHeuristics.hpp" +#include "gc/shenandoah/heuristics/shenandoahCompactOldHeuristics.hpp" #include "gc/shenandoah/heuristics/shenandoahStaticHeuristics.hpp" +#include "gc/shenandoah/heuristics/shenandoahStaticOldHeuristics.hpp" #include "gc/shenandoah/shenandoahGeneration.hpp" #include "gc/shenandoah/mode/shenandoahMode.hpp" @@ -48,3 +52,21 @@ ShenandoahHeuristics* ShenandoahMode::initialize_heuristics(ShenandoahGeneration ShouldNotReachHere(); return NULL; } + +ShenandoahOldHeuristics* ShenandoahMode::initialize_old_heuristics(ShenandoahGeneration* generation) const { + + assert(ShenandoahGCHeuristics != NULL, "ShenandoahGCHeuristics should not equal NULL"); + if (strcmp(ShenandoahGCHeuristics, "aggressive") == 0) { + return new ShenandoahAggressiveOldHeuristics(generation); + } else if (strcmp(ShenandoahGCHeuristics, "static") == 0) { + return new ShenandoahStaticOldHeuristics(generation); + } else if (strcmp(ShenandoahGCHeuristics, "adaptive") == 0) { + return new ShenandoahAdaptiveOldHeuristics(generation); + } else if (strcmp(ShenandoahGCHeuristics, "compact") == 0) { + return new ShenandoahCompactOldHeuristics(generation); + } else { + vm_exit_during_initialization("Unknown -XX:ShenandoahGCHeuristics option"); + ShouldNotReachHere(); + return NULL; + } +} diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.hpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.hpp index 6e0975eb7b479..5c74eb74d71df 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.hpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, Red Hat, Inc. All rights reserved. + * Copyright (c) 2019, 2021, Red Hat, Inc. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -31,6 +31,7 @@ class ShenandoahGeneration; class ShenandoahHeuristics; +class ShenandoahOldHeuristics; #define SHENANDOAH_CHECK_FLAG_SET(name) \ do { \ @@ -52,6 +53,7 @@ class ShenandoahMode : public CHeapObj { public: virtual void initialize_flags() const = 0; virtual ShenandoahHeuristics* initialize_heuristics(ShenandoahGeneration* generation) const; + virtual ShenandoahOldHeuristics* initialize_old_heuristics(ShenandoahGeneration* generation) const; virtual const char* name() = 0; virtual bool is_diagnostic() = 0; virtual bool is_experimental() = 0; diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.cpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.cpp index 6a1834c9d0716..f573d7a7c73fd 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.cpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.cpp @@ -25,6 +25,7 @@ #include "precompiled.hpp" #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" #include "gc/shenandoah/heuristics/shenandoahPassiveHeuristics.hpp" +#include "gc/shenandoah/heuristics/shenandoahPassiveOldHeuristics.hpp" #include "gc/shenandoah/mode/shenandoahPassiveMode.hpp" #include "logging/log.hpp" #include "logging/logTag.hpp" @@ -59,3 +60,8 @@ ShenandoahHeuristics* ShenandoahPassiveMode::initialize_heuristics(ShenandoahGen ShouldNotReachHere(); return NULL; } + +ShenandoahOldHeuristics* ShenandoahPassiveMode::initialize_old_heuristics(ShenandoahGeneration* generation) const { + assert(ShenandoahGCHeuristics != NULL, "ShenandoahGCHeuristics should not equal NULL"); + return new ShenandoahPassiveOldHeuristics(generation); +} diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.hpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.hpp index e39c0cbe8ac50..ec5e5f85f817a 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.hpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, Red Hat, Inc. All rights reserved. + * Copyright (c) 2019, 2021, Red Hat, Inc. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -31,6 +31,7 @@ class ShenandoahPassiveMode : public ShenandoahMode { public: virtual void initialize_flags() const; virtual ShenandoahHeuristics* initialize_heuristics(ShenandoahGeneration* generation) const; + virtual ShenandoahOldHeuristics* initialize_old_heuristics(ShenandoahGeneration* generation) const; virtual const char* name() { return "Passive"; } virtual bool is_diagnostic() { return true; } virtual bool is_experimental() { return false; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp index 6dee549671a2b..c7d95b12dd836 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp @@ -46,6 +46,8 @@ class ShenandoahConcurrentGC : public ShenandoahGC { private: ShenandoahConcurrentMark _mark; ShenandoahDegenPoint _degen_point; + +protected: ShenandoahGeneration* const _generation; public: diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index 86fe4fb3e935c..947526252dfd5 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -71,9 +71,7 @@ class ShenandoahResetBitmapTask : public ShenandoahHeapRegionClosure { bool is_thread_safe() { return true; } }; -void ShenandoahGeneration::initialize_heuristics(ShenandoahMode* gc_mode) { - _heuristics = gc_mode->initialize_heuristics(this); - +void ShenandoahGeneration::confirm_heuristics_mode() { if (_heuristics->is_diagnostic() && !UnlockDiagnosticVMOptions) { vm_exit_during_initialization( err_msg("Heuristics \"%s\" is diagnostic, and must be enabled via -XX:+UnlockDiagnosticVMOptions.", @@ -86,6 +84,19 @@ void ShenandoahGeneration::initialize_heuristics(ShenandoahMode* gc_mode) { } } +ShenandoahOldHeuristics* ShenandoahGeneration::initialize_old_heuristics(ShenandoahMode* gc_mode) { + ShenandoahOldHeuristics* old_heuristics = gc_mode->initialize_old_heuristics(this); + _heuristics = old_heuristics; + confirm_heuristics_mode(); + return old_heuristics; +} + +ShenandoahHeuristics* ShenandoahGeneration::initialize_heuristics(ShenandoahMode* gc_mode) { + _heuristics = gc_mode->initialize_heuristics(this); + confirm_heuristics_mode(); + return _heuristics; +} + size_t ShenandoahGeneration::bytes_allocated_since_gc_start() { return ShenandoahHeap::heap()->bytes_allocated_since_gc_start(); } @@ -148,7 +159,7 @@ void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) { ShenandoahPhaseTimings::degen_gc_choose_cset); ShenandoahHeapLocker locker(heap->lock()); heap->collection_set()->clear(); - _heuristics->choose_collection_set(heap->collection_set()); + _heuristics->choose_collection_set(heap->collection_set(), heap->old_heuristics()); } { @@ -274,4 +285,4 @@ size_t ShenandoahGeneration::available() const { size_t in_use = used(); size_t soft_capacity = soft_max_capacity(); return in_use > soft_capacity ? 0 : soft_capacity - in_use; -} \ No newline at end of file +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index bc8cf7b42bfae..56591106d5e63 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -26,7 +26,7 @@ #define SHARE_VM_GC_SHENANDOAH_SHENANDOAHGENERATION_HPP #include "memory/allocation.hpp" -#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" +#include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp" #include "gc/shenandoah/mode/shenandoahGenerationalMode.hpp" #include "gc/shenandoah/shenandoahLock.hpp" #include "gc/shenandoah/shenandoahMarkingContext.hpp" @@ -60,7 +60,9 @@ class ShenandoahGeneration : public CHeapObj { virtual const char* name() const = 0; - void initialize_heuristics(ShenandoahMode* gc_mode); + ShenandoahHeuristics* initialize_heuristics(ShenandoahMode* gc_mode); + + ShenandoahOldHeuristics* initialize_old_heuristics(ShenandoahMode* gc_mode); virtual size_t soft_max_capacity() const { return _soft_max_capacity; } virtual size_t max_capacity() const { return _max_capacity; } @@ -121,9 +123,12 @@ class ShenandoahGeneration : public CHeapObj { void increase_used(size_t bytes); void decrease_used(size_t bytes); - protected: +protected: virtual bool is_concurrent_mark_in_progress() = 0; + +private: + void confirm_heuristics_mode(); }; #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHGENERATION_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 5df3436078996..14ea40ffbb1b2 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -479,9 +479,10 @@ void ShenandoahHeap::initialize_heuristics() { _gc_mode->name())); } + // ojo: want to instantiate a ShenandoahOldHeuristics object here + _old_heuristics = _old_generation->initialize_old_heuristics(_gc_mode); _global_generation->initialize_heuristics(_gc_mode); _young_generation->initialize_heuristics(_gc_mode); - _old_generation->initialize_heuristics(_gc_mode); } #ifdef _MSC_VER @@ -492,6 +493,7 @@ void ShenandoahHeap::initialize_heuristics() { ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) : CollectedHeap(), _gc_generation(NULL), + _old_heuristics(nullptr), _initial_size(0), _used(0), _committed(0), diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index 022087da082c8..3fdee2cc29af8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -52,6 +52,7 @@ class ShenandoahGCStateResetter; class ShenandoahGeneration; class ShenandoahYoungGeneration; class ShenandoahHeuristics; +class ShenandoahOldHeuristics; class ShenandoahMarkingContext; class ShenandoahPhaseTimings; class ShenandoahHeap; @@ -146,6 +147,7 @@ class ShenandoahHeap : public CollectedHeap { private: ShenandoahHeapLock _lock; ShenandoahGeneration* _gc_generation; + ShenandoahOldHeuristics* _old_heuristics; public: ShenandoahHeapLock* lock() { @@ -161,6 +163,10 @@ class ShenandoahHeap : public CollectedHeap { _gc_generation = generation; } + ShenandoahOldHeuristics* old_heuristics() { + return _old_heuristics; + } + bool is_gc_generation_young() const; // ---------- Initialization, termination, identification, printing routines diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index fe2c8daa61c38..f7b55dabf3cef 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -403,9 +403,44 @@ void ShenandoahHeapRegion::print_on(outputStream* st) const { st->cr(); } +// oop_iterate without closure +void ShenandoahHeapRegion::oop_fill_and_coalesce() { + HeapWord* obj_addr = bottom(); + HeapWord* t = top(); + + assert(!is_humongous(), "No need to fill or coalesce humongous regions"); + if (!is_active()) return; + + ShenandoahHeap* heap = ShenandoahHeap::heap(); + ShenandoahMarkingContext* marking_context = heap->marking_context(); + + // Expect this to be invoked only from within threads perfoming old-gen GC, and expect + // old-gen marking to be completed before these threads invoke this service. + assert(heap->active_generation()->is_mark_complete(), "sanity"); + + while (obj_addr < t) { + oop obj = oop(obj_addr); + if (marking_context->is_marked(obj)) { + assert(obj->klass() != NULL, "klass should not be NULL"); + obj_addr += obj->size(); + } else { + // Object is not marked. Coalesce and fill dead object with dead neighbors. + HeapWord* next_marked_obj = marking_context->get_next_marked_addr(obj_addr, t); + assert(next_marked_obj <= t, "next marked object cannot exceed top"); + size_t fill_size = next_marked_obj - obj_addr; + ShenandoahHeap::fill_with_object(obj_addr, fill_size); + heap->card_scan()->coalesce_objects(obj_addr, fill_size); + obj_addr = next_marked_obj; + } + } +} + + void ShenandoahHeapRegion::oop_iterate(OopIterateClosure* blk, bool fill_dead_objects, bool reregister_coalesced_objects) { if (!is_active()) return; if (is_humongous()) { + // TODO: This doesn't look right. This registers objects if !reregister, and it isn't filling if fill_dead_objects. + // Furthermore, register and fill should be done after iterating. if (fill_dead_objects && !reregister_coalesced_objects) { ShenandoahHeap::heap()->card_scan()->register_object(bottom()); } @@ -431,41 +466,26 @@ void ShenandoahHeapRegion::oop_iterate_objects(OopIterateClosure* blk, bool fill ShenandoahMarkingContext* marking_context = heap->marking_context(); assert(heap->active_generation()->is_mark_complete(), "sanity"); - HeapWord* fill_addr = NULL; - size_t fill_size = 0; while (obj_addr < t) { oop obj = oop(obj_addr); if (marking_context->is_marked(obj)) { - if (fill_addr != NULL) { - if (reregister_coalesced_objects) { // change existing crossing map information - heap->card_scan()->coalesce_objects(fill_addr, fill_size); - } else { // establish new crossing map information - heap->card_scan()->register_object(fill_addr); - } - ShenandoahHeap::fill_with_object(fill_addr, fill_size); - fill_addr = NULL; - } assert(obj->klass() != NULL, "klass should not be NULL"); - if (!reregister_coalesced_objects) + if (!reregister_coalesced_objects) { heap->card_scan()->register_object(obj_addr); + } obj_addr += obj->oop_iterate_size(blk); } else { - int size = obj->size(); - if (fill_addr == NULL) { - fill_addr = obj_addr; - fill_size = size; - } else { - fill_size += size; + // Object is not marked. Coalesce and fill dead object with dead neighbors. + HeapWord* next_marked_obj = marking_context->get_next_marked_addr(obj_addr, t); + assert(next_marked_obj <= t, "next marked object cannot exceed top"); + size_t fill_size = next_marked_obj - obj_addr; + ShenandoahHeap::fill_with_object(obj_addr, fill_size); + if (reregister_coalesced_objects) { + heap->card_scan()->coalesce_objects(obj_addr, fill_size); + } else { // establish new crossing map information + heap->card_scan()->register_object(obj_addr); } - obj_addr += size; - } - } - if (fill_addr != NULL) { - ShenandoahHeap::fill_with_object(fill_addr, fill_size); - if (reregister_coalesced_objects) { // change existing crossing map information - heap->card_scan()->coalesce_objects(fill_addr, fill_size); - } else { // establish new crossing map information - heap->card_scan()->register_object(fill_addr); + obj_addr = next_marked_obj; } } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp index 23f882f0ed009..4f77892fd8bb4 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp @@ -360,6 +360,9 @@ class ShenandoahHeapRegion { void recycle(); + // coalesce contiguous spans of garbage objects by filling header and reregistering start locations with remembered set. + void oop_fill_and_coalesce(); + void oop_iterate(OopIterateClosure* cl, bool fill_dead_objects = false, bool reregister_coalesced_objects = false); void oop_iterate_humongous(OopIterateClosure* cl); @@ -411,6 +414,7 @@ class ShenandoahHeapRegion { void do_uncommit(); void oop_iterate_objects(OopIterateClosure* cl, bool fill_dead_objects, bool reregister_coalesced_objects); + void oop_iterate_objects(bool fill_dead_objects, bool reregister_coalesced_objects); inline void internal_increase_live_data(size_t s); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp index f07543d923e79..4b81b557ee26a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp @@ -26,12 +26,56 @@ #include "gc/shenandoah/shenandoahFreeSet.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" +#include "gc/shenandoah/shenandoahMonitoringSupport.hpp" #include "gc/shenandoah/shenandoahOldGC.hpp" #include "gc/shenandoah/shenandoahOopClosures.inline.hpp" +#include "gc/shenandoah/shenandoahGeneration.hpp" +#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" +#include "gc/shenandoah/shenandoahWorkerPolicy.hpp" +#include "utilities/events.hpp" + +class ShenandoahConcurrentCoalesceAndFillTask : public AbstractGangTask { +private: + // remember nworkers, coalesce_and_fill_region_array,coalesce_and_fill_regions_count + + uint _nworkers; + ShenandoahHeapRegion** _coalesce_and_fill_region_array; + uint _coalesce_and_fill_region_count; + +public: + ShenandoahConcurrentCoalesceAndFillTask(uint nworkers, + ShenandoahHeapRegion** coalesce_and_fill_region_array, uint region_count) : + AbstractGangTask("Shenandoah Concurrent Coalesce and Fill"), + _nworkers(nworkers), + _coalesce_and_fill_region_array(coalesce_and_fill_region_array), + _coalesce_and_fill_region_count(region_count) { + } + + void work(uint worker_id) { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + + for (uint region_idx = worker_id; region_idx < _coalesce_and_fill_region_count; region_idx += _nworkers) { + ShenandoahHeapRegion* r = _coalesce_and_fill_region_array[region_idx]; + if (!r->is_humongous()) + r->oop_fill_and_coalesce(); + else { + // there's only one object in this region and it's not garbage, so no need to coalesce or fill + } + } + } +}; + ShenandoahOldGC::ShenandoahOldGC(ShenandoahGeneration* generation, ShenandoahSharedFlag& allow_preemption) : ShenandoahConcurrentGC(generation), _allow_preemption(allow_preemption) {} +void ShenandoahOldGC::entry_old_evacuations() { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + ShenandoahOldHeuristics* old_heuristics = heap->old_heuristics(); + entry_coalesce_and_fill(); + old_heuristics->start_old_evacuations(); +} + bool ShenandoahOldGC::collect(GCCause::Cause cause) { ShenandoahHeap* heap = ShenandoahHeap::heap(); @@ -44,6 +88,8 @@ bool ShenandoahOldGC::collect(GCCause::Cause cause) { // Complete marking under STW vmop_entry_final_mark(); + entry_old_evacuations(); + // We aren't dealing with old generation evacuation yet. Our heuristic // should not have built a cset in final mark. assert(!heap->is_evacuation_in_progress(), "Old gen evacuations are not supported"); @@ -84,3 +130,48 @@ bool ShenandoahOldGC::collect(GCCause::Cause cause) { entry_rendezvous_roots(); return true; } + +void ShenandoahOldGC::entry_coalesce_and_fill_message(char *buf, size_t len) const { + // ShenandoahHeap* const heap = ShenandoahHeap::heap(); + jio_snprintf(buf, len, "Coalescing and filling (%s)", _generation->name()); +} + +void ShenandoahOldGC::op_coalesce_and_fill() { + ShenandoahHeap* const heap = ShenandoahHeap::heap(); + + WorkGang* workers = heap->workers(); + uint nworkers = workers->active_workers(); + + assert(_generation->generation_mode() == OLD, "Only old-GC does coalesce and fill"); + + ShenandoahOldHeuristics* old_heuristics = heap->old_heuristics(); + uint coalesce_and_fill_regions_count = old_heuristics->old_coalesce_and_fill_candidates(); + ShenandoahHeapRegion* coalesce_and_fill_region_array[coalesce_and_fill_regions_count]; + + old_heuristics->get_coalesce_and_fill_candidates(coalesce_and_fill_region_array); + ShenandoahConcurrentCoalesceAndFillTask task(nworkers, coalesce_and_fill_region_array, coalesce_and_fill_regions_count); + + + // TODO: We need to implement preemption of coalesce and fill. If young-gen wants to run while we're working on this, + // we should preempt this code and then resume it after young-gen has finished. This requires that we "remember" the state + // of each worker thread so it can be resumed where it left off. Note that some worker threads may have processed more regions + // than others at the time of preemption. + + workers->run_task(&task); +} + +void ShenandoahOldGC::entry_coalesce_and_fill() { + char msg[1024]; + ShenandoahHeap* const heap = ShenandoahHeap::heap(); + + entry_coalesce_and_fill_message(msg, sizeof(msg)); + ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::coalesce_and_fill); + + TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); + EventMark em("%s", msg); + ShenandoahWorkerScope scope(heap->workers(), + ShenandoahWorkerPolicy::calc_workers_for_conc_marking(), + "concurrent coalesce and fill"); + + op_coalesce_and_fill(); +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.hpp index b6598cda05906..a613c20920c3e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.hpp @@ -35,7 +35,11 @@ class ShenandoahOldGC : public ShenandoahConcurrentGC { ShenandoahOldGC(ShenandoahGeneration* generation, ShenandoahSharedFlag& allow_preemption); bool collect(GCCause::Cause cause); private: + void entry_old_evacuations(); + void entry_coalesce_and_fill(); ShenandoahSharedFlag& _allow_preemption; + void op_coalesce_and_fill(); + void entry_coalesce_and_fill_message(char *buf, size_t len) const; }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp index 1901c7cbdadb0..217261e723a24 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp @@ -67,6 +67,8 @@ class outputStream; f(final_mark, "Pause Final Mark (N)") \ f(finish_mark, " Finish Mark") \ SHENANDOAH_PAR_PHASE_DO(finish_mark_, " FM: ", f) \ + f(coalesce_and_fill, "Coalesce and Fill Old Dead") \ + SHENANDOAH_PAR_PHASE_DO(coalesce_and_fill_, " CFOD: ", f) \ f(purge, " System Purge") \ SHENANDOAH_PAR_PHASE_DO(purge_cu_par_, " CU: ", f) \ f(purge_weak_par, " Weak Roots") \ diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.cpp index a80a890e4f1e1..cac4a3384d2fb 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.cpp @@ -43,7 +43,7 @@ ShenandoahRegulatorThread::ShenandoahRegulatorThread(ShenandoahControlThread* co _last_sleep_adjust_time(os::elapsedTime()) { ShenandoahHeap* heap = ShenandoahHeap::heap(); - _old_heuristics = get_heuristics(heap->old_generation()); + _old_heuristics = heap->old_heuristics(); _young_heuristics = get_heuristics(heap->young_generation()); _global_heuristics = get_heuristics(heap->global_generation()); @@ -134,7 +134,7 @@ void ShenandoahRegulatorThread::regulator_sleep() { } bool ShenandoahRegulatorThread::start_old_cycle() { - return _old_heuristics->should_start_gc() && _control_thread->request_concurrent_gc(OLD); + return !_old_heuristics->should_defer_gc() && _old_heuristics->should_start_gc() && _control_thread->request_concurrent_gc(OLD); } bool ShenandoahRegulatorThread::start_young_cycle() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.hpp b/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.hpp index af964ddf90989..fef7199e31340 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.hpp @@ -27,6 +27,7 @@ #include "gc/shared/concurrentGCThread.hpp" #include "gc/shared/gcCause.hpp" #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" +#include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp" #include "gc/shenandoah/shenandoahHeap.hpp" #include "runtime/mutex.hpp" @@ -74,7 +75,7 @@ class ShenandoahRegulatorThread: public ConcurrentGCThread { ShenandoahSharedFlag _heap_changed; ShenandoahControlThread* _control_thread; ShenandoahHeuristics* _young_heuristics; - ShenandoahHeuristics* _old_heuristics; + ShenandoahOldHeuristics* _old_heuristics; ShenandoahHeuristics* _global_heuristics; int _sleep; diff --git a/test/hotspot/jtreg/gc/shenandoah/generational/TestConcurrentEvac.java b/test/hotspot/jtreg/gc/shenandoah/generational/TestConcurrentEvac.java new file mode 100644 index 0000000000000..4ffa2394c9859 --- /dev/null +++ b/test/hotspot/jtreg/gc/shenandoah/generational/TestConcurrentEvac.java @@ -0,0 +1,209 @@ +/* + * Copyright (c) 2021, Amazon, Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +package gc.shenandoah.generational; + +import sun.hotspot.WhiteBox; +import java.util.Random; +import java.util.HashMap; + +/* WARNING! + * As of the date on which this test was added into the jtreg suite, heuristic + * of old-gen GC passes is very simplistic. A further shortcoming of the + * Generational Shenandoah as of introduction of this test is that it does + * not currently support full GC. If garbage collection falls behind mutator + * allocations, a full GC will be triggered and Generational Shenandoah will + * abort itself with an assertion error. Both of these limitations will be + * addressed in future releases of Generational Shenandoah. + * + * To avoid the risk of false regressions identified by this test, the heap + * size is set artificially high. Though this test is known to run reliably + * in 66 MB heap, the heap size for this test run is currently set to 256 MB. + */ + +/* + * @test TestConcurrentEvac + * @requires vm.gc.Shenandoah + * @summary Confirm that card marking and remembered set scanning do not crash. + * @library /testlibrary /test/lib / + * @build sun.hotspot.WhiteBox + * @run driver ClassFileInstaller sun.hotspot.WhiteBox + * @run main/othervm -Xbootclasspath/a:. + * -Xms256m -Xmx256m + * -XX:+IgnoreUnrecognizedVMOptions + * -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * -XX:NewRatio=1 -XX:+UnlockExperimentalVMOptions + * -XX:ShenandoahGuaranteedGCInterval=3000 + * -XX:-UseDynamicNumberOfGCThreads -XX:-ShenandoahPacing + * gc.shenandoah.generational.TestConcurrentEvac + */ + +public class TestConcurrentEvac { + private static WhiteBox wb = WhiteBox.getWhiteBox(); + + static private final int SeedForRandom = 46; + // Sequence of random numbers should end with same value + + // Smaller table will cause creation of more old-gen garbage + // as previous entries in table are overwritten with new values. + static private final int TableSize = 53; + static private final int MaxStringLength = 47; + static private final int SentenceLength = 5; + + static private Random random = new Random(SeedForRandom); + + public static class Node { + static private final int NeighborCount = 48; + static private final int ChildOverwriteCount = 32; + static private final int IntArraySize = 128; + + private String name; + + // Each Node instance holds an array containing all substrings of + // its name + + // This array has entries from 0 .. (name.length() - 1). + // num_substrings[i] represents the number of substrings that + // correspond to a name of length i+1. + private static int [] num_substrings; + + static { + // Initialize num_substrings. + // For a name of length N, there are + // N substrings of length 1 + // N-1 substrings of length 2 + // N-2 substrings of length 3 + // ... + // 1 substring of length N + // Note that: + // num_substrings[0] = 1 + // num_substrings[1] = 3 + // num_substrings[i] = (i+1)+num_substrings[i-1] + + num_substrings = new int[MaxStringLength]; + num_substrings[0] = 1; + for (int i = 1; i < MaxStringLength; i++) + num_substrings[i] = (i+1)+num_substrings[i-1]; + } + + private String [] substrings; + private Node [] neighbors; + + public Node(String name) { + this.name = name; + this.substrings = new String[num_substrings[name.length() - 1]]; + + int index = 0; + for (int substring_length = 1; + substring_length <= name.length(); substring_length++) { + for (int offset = 0; + offset + substring_length <= name.length(); offset++) { + this.substrings[index++] = name.substring(offset, + offset + substring_length); + } + } + } + + public String value() { + return name; + } + + public String arbitrary_substring() { + int index = TestConcurrentEvac.randomUnsignedInt() % substrings.length; + return substrings[index]; + } + } + + + // Return random int between 1 and MaxStringLength inclusive + static int randomStringLength() { + int length = randomUnsignedInt(); + length %= (MaxStringLength - 1); + length += 1; + return length; + } + + static String randomCharacter() { + int index = randomUnsignedInt() % 52; + return ("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ". + substring(index, index+1)); + } + + static String randomString() { + int length = randomStringLength(); + String result = new String(); // make the compiler work for this garbage... + for (int i = 0; i < length; i++) + result += randomCharacter(); + return result; + } + + static int randomUnsignedInt() { + int result = random.nextInt(); + if (result < 0) result = -result; + if (result < 0) result = 0; + return result; + } + + static int randomIndex() { + int index = randomUnsignedInt(); + index %= TableSize; + return index; + } + + public static void main(String args[]) throws Exception { + HashMap table = new HashMap(TableSize); + + if (!wb.getBooleanVMFlag("UseShenandoahGC") || + !wb.getStringVMFlag("ShenandoahGCMode").equals("generational")) + throw new IllegalStateException("Command-line options not honored!"); + + for (int count = java.lang.Integer.MAX_VALUE/1024; count >= 0; count--) { + int index = randomIndex(); + String name = randomString(); + table.put(index, new Node(name)); + } + + String conclusion = ""; + + for (int i = 0; i < SentenceLength; i++) { + Node a_node = table.get(randomIndex()); + if (a_node == null) + i--; + else { + String a_string = a_node.arbitrary_substring(); + conclusion += a_string; + conclusion += " "; + } + } + conclusion = conclusion.substring(0, conclusion.length() - 1); + + System.out.println("Conclusion is [" + conclusion + "]"); + + if (!conclusion.equals("cTy cTykJ kAkKAOWYEHbxFCmRIlyk xjYMdNmtAQXNGdIc sqHKsWnJIP")) + throw new IllegalStateException("Random sequence of words did not end well!"); + + } +} + From 816c1a611e6bd245904da56b67cb6432cb32b569 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Sat, 1 May 2021 16:39:41 +0200 Subject: [PATCH 034/254] Fix jcheck conf to point to Shenandoah project, instead of JDK project --- .jcheck/conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jcheck/conf b/.jcheck/conf index b984b5cd521c2..38aaf8e6387c9 100644 --- a/.jcheck/conf +++ b/.jcheck/conf @@ -1,5 +1,5 @@ [general] -project=jdk +project=shenandoah jbs=JDK [checks] From 9fc8ac201745b21eab1ccf36a18e90f5f0a56af7 Mon Sep 17 00:00:00 2001 From: Aleksey Shipilev Date: Mon, 3 May 2021 12:23:01 +0000 Subject: [PATCH 035/254] Fix build failures due to undeclared ShenandoahOldHeuristics Reviewed-by: rkennke, zgu --- .../share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp index 1c9e1c1f07ac0..3cebd71feda4f 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp @@ -57,6 +57,7 @@ class ShenandoahCollectionSet; class ShenandoahHeapRegion; class ShenandoahGeneration; +class ShenandoahOldHeuristics; class ShenandoahHeuristics : public CHeapObj { static const intx Concurrent_Adjust = -1; // recover from penalties From fdd30f5c605661d603374326619989def65865c7 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Mon, 3 May 2021 16:10:08 +0000 Subject: [PATCH 036/254] Fix barrier set cast for aarch64 Reviewed-by: zgu, rkennke --- .../gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp | 5 ++--- .../x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp index ee391c86d4714..2c74e2251b916 100644 --- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp @@ -626,9 +626,8 @@ void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssemb return; } - BarrierSet* bs = BarrierSet::barrier_set(); - CardTableBarrierSet* ctbs = barrier_set_cast(bs); - CardTable* ct = ctbs->card_table(); + ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set(); + CardTable* ct = bs->card_table(); Label L_loop, L_done; const Register end = count; diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp index adc0c18f9294a..96d9a42a3e40a 100644 --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp @@ -916,9 +916,8 @@ void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssemb return; } - BarrierSet *bs = BarrierSet::barrier_set(); - ShenandoahBarrierSet* ctbs = barrier_set_cast(bs); - CardTable* ct = ctbs->card_table(); + ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set(); + CardTable* ct = bs->card_table(); intptr_t disp = (intptr_t) ct->byte_map_base(); Label L_loop, L_done; From 234053e5194462a8aaf324243f8954591ef934ff Mon Sep 17 00:00:00 2001 From: Bernd Mathiske Date: Mon, 3 May 2021 22:21:35 +0000 Subject: [PATCH 037/254] Use PLABs for old gen allocations, including promotions. Reviewed-by: rkennke --- src/hotspot/share/gc/shared/plab.hpp | 4 + .../gc/shenandoah/shenandoahAllocRequest.hpp | 12 +- .../gc/shenandoah/shenandoahBarrierSet.cpp | 5 +- .../share/gc/shenandoah/shenandoahFreeSet.cpp | 82 ++++++------- .../share/gc/shenandoah/shenandoahFreeSet.hpp | 1 + .../share/gc/shenandoah/shenandoahFullGC.cpp | 2 +- .../share/gc/shenandoah/shenandoahHeap.cpp | 113 ++++++++++++++++-- .../share/gc/shenandoah/shenandoahHeap.hpp | 7 ++ .../gc/shenandoah/shenandoahHeap.inline.hpp | 90 ++++++++++---- .../gc/shenandoah/shenandoahHeapRegion.cpp | 24 +++- .../gc/shenandoah/shenandoahHeapRegion.hpp | 6 +- .../shenandoahHeapRegion.inline.hpp | 9 +- .../shenandoahHeapRegionCounters.cpp | 1 + .../shenandoahHeapRegionCounters.hpp | 3 +- .../shenandoah/shenandoahThreadLocalData.hpp | 21 ++++ .../gc/shenandoah/shenandoahVerifier.cpp | 7 +- .../gc/shenandoah/shenandoah_globals.hpp | 4 + 17 files changed, 296 insertions(+), 95 deletions(-) diff --git a/src/hotspot/share/gc/shared/plab.hpp b/src/hotspot/share/gc/shared/plab.hpp index 6180c635187fc..707874173e3f1 100644 --- a/src/hotspot/share/gc/shared/plab.hpp +++ b/src/hotspot/share/gc/shared/plab.hpp @@ -140,6 +140,10 @@ class PLAB: public CHeapObj { // Fills in the unallocated portion of the buffer with a garbage object and updates // statistics. To be called during GC. void retire(); + + HeapWord* top() { + return _top; + } }; // PLAB book-keeping. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahAllocRequest.hpp b/src/hotspot/share/gc/shenandoah/shenandoahAllocRequest.hpp index f32273122f072..e210666348e77 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahAllocRequest.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahAllocRequest.hpp @@ -32,9 +32,10 @@ class ShenandoahAllocRequest : StackObj { public: enum Type { _alloc_shared, // Allocate common, outside of TLAB - _alloc_shared_gc, // Allocate common, outside of GCLAB + _alloc_shared_gc, // Allocate common, outside of GCLAB/PLAB _alloc_tlab, // Allocate TLAB _alloc_gclab, // Allocate GCLAB + _alloc_plab, // Allocate PLAB _ALLOC_LIMIT }; @@ -48,6 +49,8 @@ class ShenandoahAllocRequest : StackObj { return "TLAB"; case _alloc_gclab: return "GCLAB"; + case _alloc_plab: + return "PLAB"; default: ShouldNotReachHere(); return ""; @@ -81,6 +84,10 @@ class ShenandoahAllocRequest : StackObj { return ShenandoahAllocRequest(min_size, requested_size, _alloc_gclab, ShenandoahRegionAffiliation::YOUNG_GENERATION); } + static inline ShenandoahAllocRequest for_plab(size_t min_size, size_t requested_size) { + return ShenandoahAllocRequest(min_size, requested_size, _alloc_plab, ShenandoahRegionAffiliation::OLD_GENERATION); + } + static inline ShenandoahAllocRequest for_shared_gc(size_t requested_size, ShenandoahRegionAffiliation affiliation) { return ShenandoahAllocRequest(0, requested_size, _alloc_shared_gc, affiliation); } @@ -125,6 +132,7 @@ class ShenandoahAllocRequest : StackObj { case _alloc_shared: return true; case _alloc_gclab: + case _alloc_plab: case _alloc_shared_gc: return false; default: @@ -139,6 +147,7 @@ class ShenandoahAllocRequest : StackObj { case _alloc_shared: return false; case _alloc_gclab: + case _alloc_plab: case _alloc_shared_gc: return true; default: @@ -151,6 +160,7 @@ class ShenandoahAllocRequest : StackObj { switch (_alloc_type) { case _alloc_tlab: case _alloc_gclab: + case _alloc_plab: return true; case _alloc_shared: case _alloc_shared_gc: diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp index ff7cd374d9c24..d26eeb48695af 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp @@ -120,8 +120,11 @@ void ShenandoahBarrierSet::on_thread_detach(Thread *thread) { gclab->retire(); } - // SATB protocol requires to keep alive reacheable oops from roots at the beginning of GC ShenandoahHeap* const heap = ShenandoahHeap::heap(); + PLAB* plab = ShenandoahThreadLocalData::plab(thread); + heap->retire_plab(plab); + + // SATB protocol requires to keep alive reacheable oops from roots at the beginning of GC if (heap->is_concurrent_mark_in_progress()) { ShenandoahKeepAliveClosure oops; StackWatermarkSet::finish_processing(thread->as_Java_thread(), &oops, StackWatermarkKind::gc); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index eadb51ff644d9..2868f9df43396 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -35,7 +35,6 @@ #include "memory/resourceArea.hpp" #include "runtime/orderAccess.hpp" - ShenandoahFreeSet::ShenandoahFreeSet(ShenandoahHeap* heap, size_t max_regions) : _heap(heap), _mutator_free_bitmap(max_regions, mtGC), @@ -65,6 +64,23 @@ bool ShenandoahFreeSet::is_collector_free(size_t idx) const { return _collector_free_bitmap.at(idx); } +HeapWord* ShenandoahFreeSet::allocate_with_affiliation(ShenandoahRegionAffiliation affiliation, ShenandoahAllocRequest& req, bool& in_new_region) { + for (size_t c = _collector_rightmost + 1; c > _collector_leftmost; c--) { + // size_t is unsigned, need to dodge underflow when _leftmost = 0 + size_t idx = c - 1; + if (is_collector_free(idx)) { + ShenandoahHeapRegion* r = _heap->get_region(idx); + if (r->affiliation() == affiliation) { + HeapWord* result = try_allocate_in(r, req, in_new_region); + if (result != NULL) { + return result; + } + } + } + } + return NULL; +} + HeapWord* ShenandoahFreeSet::allocate_single(ShenandoahAllocRequest& req, bool& in_new_region) { // Scan the bitmap looking for a first fit. // @@ -96,34 +112,17 @@ HeapWord* ShenandoahFreeSet::allocate_single(ShenandoahAllocRequest& req, bool& break; } case ShenandoahAllocRequest::_alloc_gclab: + case ShenandoahAllocRequest::_alloc_plab: case ShenandoahAllocRequest::_alloc_shared_gc: { - // size_t is unsigned, need to dodge underflow when _leftmost = 0 - - // Fast-path: try to allocate in the collector view first - for (size_t c = _collector_rightmost + 1; c > _collector_leftmost; c--) { - size_t idx = c - 1; - if (is_collector_free(idx)) { - ShenandoahHeapRegion* r = _heap->get_region(idx); - if (r->is_young() && req.is_old()) { - // We don't want to cannibalize a young region to satisfy - // an evacuation from an old region. - continue; - } - HeapWord* result = try_allocate_in(r, req, in_new_region); - if (result != NULL) { - if (r->is_old()) { - // HEY! This is a very coarse card marking. We hope to repair - // such cards during remembered set scanning. - - // HEY! To support full generality with alternative remembered set implementations, - // is preferable to not make direct access to the current card_table implementation. - // Try ShenandoahHeap::heap()->card_scan()->mark_range_as_dirty(result, req.actual_size()); - - ShenandoahBarrierSet::barrier_set()->card_table()->dirty_MemRegion(MemRegion(result, req.actual_size())); - } - return result; - } - } + // First try to fit into a region that is already in use in the same generation. + HeapWord* result = allocate_with_affiliation(req.affiliation(), req, in_new_region); + if (result != NULL) { + return result; + } + // Then try a free region that is dedicated to GC allocations. + result = allocate_with_affiliation(FREE, req, in_new_region); + if (result != NULL) { + return result; } // No dice. Can we borrow space from mutator view? @@ -131,29 +130,15 @@ HeapWord* ShenandoahFreeSet::allocate_single(ShenandoahAllocRequest& req, bool& return NULL; } - // Try to steal the empty region from the mutator view + // Try to steal an empty region from the mutator view. for (size_t c = _mutator_rightmost + 1; c > _mutator_leftmost; c--) { size_t idx = c - 1; if (is_mutator_free(idx)) { ShenandoahHeapRegion* r = _heap->get_region(idx); if (can_allocate_from(r)) { - if (r->is_young() && req.is_old()) { - continue; - } - flip_to_gc(r); HeapWord *result = try_allocate_in(r, req, in_new_region); if (result != NULL) { - if (r->is_old()) { - // HEY! This is a very coarse card marking. We hope to repair - // such cards during remembered set scanning. - - // HEY! To support full generality with alternative remembered set implementations, - // is preferable to not make direct access to the current card_table implementation. - // Try ShenandoahHeap::heap()->card_scan()->mark_range_as_dirty(result, req.actual_size()); - - ShenandoahBarrierSet::barrier_set()->card_table()->dirty_MemRegion(MemRegion(result, req.actual_size())); - } return result; } } @@ -163,13 +148,11 @@ HeapWord* ShenandoahFreeSet::allocate_single(ShenandoahAllocRequest& req, bool& // No dice. Do not try to mix mutator and GC allocations, because // URWM moves due to GC allocations would expose unparsable mutator // allocations. - break; } default: ShouldNotReachHere(); } - return NULL; } @@ -202,11 +185,11 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah size = free; } if (size >= req.min_size()) { - result = r->allocate(size, req.type()); + result = r->allocate(size, req); assert (result != NULL, "Allocation must succeed: free " SIZE_FORMAT ", actual " SIZE_FORMAT, free, size); } } else { - result = r->allocate(size, req.type()); + result = r->allocate(size, req); } if (result != NULL) { @@ -439,6 +422,10 @@ void ShenandoahFreeSet::flip_to_gc(ShenandoahHeapRegion* r) { adjust_bounds(); } assert_bounds(); + + // We do not ensure that the region is no longer trash, + // relying on try_allocate_in(), which always comes next, + // to recycle trash before attempting to allocate anything in the region. } void ShenandoahFreeSet::clear() { @@ -599,6 +586,7 @@ HeapWord* ShenandoahFreeSet::allocate(ShenandoahAllocRequest& req, bool& in_new_ case ShenandoahAllocRequest::_alloc_shared_gc: in_new_region = true; return allocate_contiguous(req); + case ShenandoahAllocRequest::_alloc_plab: case ShenandoahAllocRequest::_alloc_gclab: case ShenandoahAllocRequest::_alloc_tlab: in_new_region = false; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp index 634adfb63e034..a1a8a9634546f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp @@ -49,6 +49,7 @@ class ShenandoahFreeSet : public CHeapObj { bool is_collector_free(size_t idx) const; HeapWord* try_allocate_in(ShenandoahHeapRegion* region, ShenandoahAllocRequest& req, bool& in_new_region); + HeapWord* allocate_with_affiliation(ShenandoahRegionAffiliation affiliation, ShenandoahAllocRequest& req, bool& in_new_region); HeapWord* allocate_single(ShenandoahAllocRequest& req, bool& in_new_region); HeapWord* allocate_contiguous(ShenandoahAllocRequest& req); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index 0f21734fa6aaa..19f48700f1ac9 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -331,7 +331,7 @@ class ShenandoahPrepareForCompactionObjectClosure : public ObjectClosure { void finish_region() { assert(_to_region != NULL, "should not happen"); if (_heap->mode()->is_generational() && _to_region->affiliation() == FREE) { - // HEY! Changing this region to young during compaction may not be + // TODO: Changing this region to young during compaction may not be // technically correct here because it completely disregards the ages // and origins of the objects being moved. It is, however, certainly // more correct than putting live objects into a region without a diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 14ea40ffbb1b2..b757379be4d68 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -833,6 +833,67 @@ HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) return gclab->allocate(size); } +HeapWord* ShenandoahHeap::allocate_from_plab_slow(Thread* thread, size_t size) { + // New object should fit the PLAB size + size_t min_size = MAX2(size, PLAB::min_size()); + + // Figure out size of new PLAB, looking back at heuristics. Expand aggressively. + size_t new_size = ShenandoahThreadLocalData::plab_size(thread) * 2; + new_size = MIN2(new_size, PLAB::max_size()); + new_size = MAX2(new_size, PLAB::min_size()); + + // Record new heuristic value even if we take any shortcut. This captures + // the case when moderately-sized objects always take a shortcut. At some point, + // heuristics should catch up with them. + ShenandoahThreadLocalData::set_plab_size(thread, new_size); + + if (new_size < size) { + // New size still does not fit the object. Fall back to shared allocation. + // This avoids retiring perfectly good PLABs, when we encounter a large object. + return NULL; + } + + // Retire current PLAB, and allocate a new one. + PLAB* plab = ShenandoahThreadLocalData::plab(thread); + retire_plab(plab); + + size_t actual_size = 0; + HeapWord* plab_buf = allocate_new_plab(min_size, new_size, &actual_size); + if (plab_buf == NULL) { + return NULL; + } + + assert (size <= actual_size, "allocation should fit"); + + if (ZeroTLAB) { + // ..and clear it. + Copy::zero_to_words(plab_buf, actual_size); + } else { + // ...and zap just allocated object. +#ifdef ASSERT + // Skip mangling the space corresponding to the object header to + // ensure that the returned space is not considered parsable by + // any concurrent GC thread. + size_t hdr_size = oopDesc::header_size(); + Copy::fill_to_words(plab_buf + hdr_size, actual_size - hdr_size, badHeapWordVal); +#endif // ASSERT + } + plab->set_buf(plab_buf, actual_size); + return plab->allocate(size); +} + +void ShenandoahHeap::retire_plab(PLAB* plab) { + size_t waste = plab->waste(); + HeapWord* top = plab->top(); + plab->retire(); + if (top != NULL && plab->waste() > waste) { + // If retiring the plab created a filler object, then we + // need to register it with our card scanner so it can + // safely walk the region backing the plab. + card_scan()->register_object(top); + } +} + HeapWord* ShenandoahHeap::allocate_new_tlab(size_t min_size, size_t requested_size, size_t* actual_size) { @@ -859,6 +920,19 @@ HeapWord* ShenandoahHeap::allocate_new_gclab(size_t min_size, return res; } +HeapWord* ShenandoahHeap::allocate_new_plab(size_t min_size, + size_t word_size, + size_t* actual_size) { + ShenandoahAllocRequest req = ShenandoahAllocRequest::for_plab(min_size, word_size); + HeapWord* res = allocate_memory(req); + if (res != NULL) { + *actual_size = req.actual_size(); + } else { + *actual_size = 0; + } + return res; +} + HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) { intptr_t pacer_epoch = 0; bool in_new_region = false; @@ -954,23 +1028,25 @@ HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req // // The thread allocating b and the thread allocating c can "race" in various ways, resulting in confusion, such as last-start // representing object b while first-start represents object c. This is why we need to require all register_object() - // invocations to be "mutually exclusive". Later, when we use GCLABs to allocate memory for promotions and evacuations, + // invocations to be "mutually exclusive". Later, when we use GCLABs and PLABs to allocate memory for promotions and evacuations, // the protocol may work something like the following: - // 1. The GCLAB is allocated by this (or similar) function, while holding the global lock. - // 2. The GCLAB is registered as a single object. - /// 3. The GCLAB is always aligned at the start of a card memory range and is always a multiple of the card-table memory range size - // 3. Individual allocations carved from the GCLAB are not immediately registered - // 4. When the GCLAB is eventually retired, all of the objects allocated within the GCLAB are registered in batch by a - // single thread. No further synchronization is required because no other allocations will pertain to the same + // 1. The GCLAB/PLAB is allocated by this (or similar) function, while holding the global lock. + // 2. The GCLAB/PLAB is always aligned at the start of a card memory range + // and is always a multiple of the card-table memory range size. + // 3. Individual allocations carved from a GCLAB/PLAB are not immediately registered. + // 4. A PLAB is registered as a single object. + // 5. When a PLAB is eventually retired, all of the objects allocated within the GCLAB/PLAB are registered in batch by a + // single thread. No further synchronization is required because no other allocations will pertain to the same // card-table memory ranges. // - // The other case that needs special handling is promotion of regions en masse. When the region is promoted, all objects contained - // within the region are registered. Since the region is a multiple of card-table memory range sizes, there is no need for - // synchronization. It might be nice to figure out how to allow multiple threads to work together to register all of the objects in - // a promoted region, or at least try to balance the efforts so that different gc threads work on registering the objects of - // different heap regions. But that effort will come later. + // The other case that needs special handling is region promotion. When a region is promoted, all objects contained + // in it are registered. Since the region is a multiple of card table memory range sizes, there is no need for + // synchronization. + // TODO: figure out how to allow multiple threads to work together to register all of the objects in + // a promoted region, or at least try to balance the efforts so that different GC threads work + // on registering the objects of different heap regions. // - if (result != NULL && req.affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) { + if (mode()->is_generational() && result != NULL && req.affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) { ShenandoahHeap::heap()->card_scan()->register_object(result); } return result; @@ -1140,6 +1216,10 @@ class ShenandoahCheckCleanGCLABClosure : public ThreadClosure { PLAB* gclab = ShenandoahThreadLocalData::gclab(thread); assert(gclab != NULL, "GCLAB should be initialized for %s", thread->name()); assert(gclab->words_remaining() == 0, "GCLAB should not need retirement"); + + PLAB* plab = ShenandoahThreadLocalData::plab(thread); + assert(plab != NULL, "PLAB should be initialized for %s", thread->name()); + assert(plab->words_remaining() == 0, "PLAB should not need retirement"); } }; @@ -1155,6 +1235,13 @@ class ShenandoahRetireGCLABClosure : public ThreadClosure { if (_resize && ShenandoahThreadLocalData::gclab_size(thread) > 0) { ShenandoahThreadLocalData::set_gclab_size(thread, 0); } + + PLAB* plab = ShenandoahThreadLocalData::plab(thread); + assert(plab != NULL, "PLAB should be initialized for %s", thread->name()); + ShenandoahHeap::heap()->retire_plab(plab); + if (_resize && ShenandoahThreadLocalData::plab_size(thread) > 0) { + ShenandoahThreadLocalData::set_plab_size(thread, 0); + } } }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index 3fdee2cc29af8..c648fdb517c47 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -44,6 +44,7 @@ class ConcurrentGCTimer; class ObjectIterateScanRootClosure; +class PLAB; class ShenandoahCollectorPolicy; class ShenandoahControlThread; class ShenandoahRegulatorThread; @@ -567,10 +568,15 @@ class ShenandoahHeap : public CollectedHeap { // private: HeapWord* allocate_memory_under_lock(ShenandoahAllocRequest& request, bool& in_new_region); + inline HeapWord* allocate_from_gclab(Thread* thread, size_t size); HeapWord* allocate_from_gclab_slow(Thread* thread, size_t size); HeapWord* allocate_new_gclab(size_t min_size, size_t word_size, size_t* actual_size); + inline HeapWord* allocate_from_plab(Thread* thread, size_t size); + HeapWord* allocate_from_plab_slow(Thread* thread, size_t size); + HeapWord* allocate_new_plab(size_t min_size, size_t word_size, size_t* actual_size); + public: HeapWord* allocate_memory(ShenandoahAllocRequest& request); HeapWord* mem_allocate(size_t size, bool* what); @@ -676,6 +682,7 @@ class ShenandoahHeap : public CollectedHeap { inline RememberedScanner* card_scan() { return _card_scan; } void clear_cards_for(ShenandoahHeapRegion* region); void mark_card_as_dirty(HeapWord* location); + void retire_plab(PLAB* plab); // ---------- Helper functions // diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp index 21a7fa51b19aa..19231813d43ae 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp @@ -42,6 +42,7 @@ #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp" #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp" #include "gc/shenandoah/shenandoahThreadLocalData.hpp" +#include "gc/shenandoah/shenandoahScanRemembered.inline.hpp" #include "gc/shenandoah/mode/shenandoahMode.hpp" #include "oops/compressedOops.inline.hpp" #include "oops/oop.inline.hpp" @@ -211,10 +212,31 @@ inline HeapWord* ShenandoahHeap::allocate_from_gclab(Thread* thread, size_t size if (obj != NULL) { return obj; } - // Otherwise... return allocate_from_gclab_slow(thread, size); } +inline HeapWord* ShenandoahHeap::allocate_from_plab(Thread* thread, size_t size) { + assert(UseTLAB, "TLABs should be enabled"); + + PLAB* plab = ShenandoahThreadLocalData::plab(thread); + if (plab == NULL) { + assert(!thread->is_Java_thread() && !thread->is_Worker_thread(), + "Performance: thread should have PLAB: %s", thread->name()); + // No PLABs in this thread, fallback to shared allocation + return NULL; + } + HeapWord* obj = plab->allocate(size); + if (obj == NULL) { + obj = allocate_from_plab_slow(thread, size); + } + + if (mode()->is_generational() && obj != NULL) { + ShenandoahHeap::heap()->card_scan()->register_object(obj); + } + + return obj; +} + inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) { if (ShenandoahThreadLocalData::is_oom_during_evac(Thread::current())) { // This thread went through the OOM during evac protocol and it is safe to return @@ -241,13 +263,6 @@ inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) { } else if (mark.age() >= InitialTenuringThreshold) { oop result = try_evacuate_object(p, thread, r, OLD_GENERATION); if (result != NULL) { - // TODO: Just marking the cards covering this object dirty - // may overall be less efficient than scanning it now for references to young gen - // or other alternatives like deferred card marking or scanning. - // We should revisit this. - // Furthermore, the object start should be registered for remset scanning. - MemRegion mr(cast_from_oop(result), result->size()); - ShenandoahBarrierSet::barrier_set()->card_table()->invalidate(mr); return result; } } @@ -256,7 +271,7 @@ inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) { } inline oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region, ShenandoahRegionAffiliation target_gen) { - bool alloc_from_gclab = true; + bool alloc_from_lab = true; HeapWord* copy = NULL; size_t size = p->size(); @@ -266,13 +281,28 @@ inline oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, Shenandoah copy = NULL; } else { #endif - if (UseTLAB && target_gen == YOUNG_GENERATION) { - copy = allocate_from_gclab(thread, size); + if (UseTLAB) { + switch (target_gen) { + case YOUNG_GENERATION: { + copy = allocate_from_gclab(thread, size); + break; + } + case OLD_GENERATION: { + if (ShenandoahUsePLAB) { + copy = allocate_from_plab(thread, size); + } + break; + } + default: { + ShouldNotReachHere(); + break; + } + } } if (copy == NULL) { ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared_gc(size, target_gen); copy = allocate_memory(req); - alloc_from_gclab = false; + alloc_from_lab = false; } #ifdef ASSERT } @@ -311,6 +341,10 @@ inline oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, Shenandoah // Try to install the new forwarding pointer. oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val); if (result == copy_val) { + if (target_gen == OLD_GENERATION) { + ShenandoahBarrierSet::barrier_set()->card_table()->dirty_MemRegion(MemRegion(copy, size)); + card_scan()->register_object(copy); + } // Successfully evacuated. Our copy is now the public one! shenandoah_assert_correct(NULL, copy_val); return copy_val; @@ -320,17 +354,33 @@ inline oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, Shenandoah // But if it happens to contain references to evacuated regions, those references would // not get updated for this stale copy during this cycle, and we will crash while scanning // it the next cycle. - // - // For GCLAB allocations, it is enough to rollback the allocation ptr. Either the next - // object will overwrite this stale copy, or the filler object on LAB retirement will - // do this. For non-GCLAB allocations, we have no way to retract the allocation, and - // have to explicitly overwrite the copy with the filler object. With that overwrite, - // we have to keep the fwdptr initialized and pointing to our (stale) copy. - if (alloc_from_gclab) { - ShenandoahThreadLocalData::gclab(thread)->undo_allocation(copy, size); + if (alloc_from_lab) { + // For LAB allocations, it is enough to rollback the allocation ptr. Either the next + // object will overwrite this stale copy, or the filler object on LAB retirement will + // do this. + switch (target_gen) { + case YOUNG_GENERATION: { + ShenandoahThreadLocalData::gclab(thread)->undo_allocation(copy, size); + break; + } + case OLD_GENERATION: { + ShenandoahThreadLocalData::plab(thread)->undo_allocation(copy, size); + break; + } + default: { + ShouldNotReachHere(); + break; + } + } } else { + // For non-LAB allocations, we have no way to retract the allocation, and + // have to explicitly overwrite the copy with the filler object. With that overwrite, + // we have to keep the fwdptr initialized and pointing to our (stale) copy. fill_with_object(copy, size); shenandoah_assert_correct(NULL, copy_val); + if (target_gen == OLD_GENERATION) { + card_scan()->register_object(copy); + } } shenandoah_assert_correct(NULL, result); return result; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index f7b55dabf3cef..338dee4d72fdf 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -70,10 +70,11 @@ ShenandoahHeapRegion::ShenandoahHeapRegion(HeapWord* start, size_t index, bool c _top(start), _tlab_allocs(0), _gclab_allocs(0), + _plab_allocs(0), _live_data(0), _critical_pins(0), _update_watermark(start), - _affiliation(ShenandoahRegionAffiliation::FREE), + _affiliation(FREE), _age(0) { assert(Universe::on_page_boundary(_bottom) && Universe::on_page_boundary(_end), @@ -91,13 +92,14 @@ void ShenandoahHeapRegion::report_illegal_transition(const char *method) { fatal("%s", ss.as_string()); } -void ShenandoahHeapRegion::make_regular_allocation() { +void ShenandoahHeapRegion::make_regular_allocation(ShenandoahRegionAffiliation affiliation) { shenandoah_assert_heaplocked(); reset_age(); switch (_state) { case _empty_uncommitted: do_commit(); case _empty_committed: + set_affiliation(affiliation); set_state(_regular); case _regular: case _pinned: @@ -119,6 +121,12 @@ void ShenandoahHeapRegion::make_regular_bypass() { case _cset: case _humongous_start: case _humongous_cont: + // TODO: Changing this region to young during compaction may not be + // technically correct here because it completely disregards the ages + // and origins of the objects being moved. It is, however, certainly + // more correct than putting live objects into a region without a + // generational affiliation. + set_affiliation(YOUNG_GENERATION); set_state(_regular); return; case _pinned_cset: @@ -220,6 +228,7 @@ void ShenandoahHeapRegion::make_unpinned() { switch (_state) { case _pinned: + assert(affiliation() != FREE, "Pinned region should not be FREE"); set_state(_regular); return; case _regular: @@ -318,10 +327,11 @@ void ShenandoahHeapRegion::make_committed_bypass() { void ShenandoahHeapRegion::reset_alloc_metadata() { _tlab_allocs = 0; _gclab_allocs = 0; + _plab_allocs = 0; } size_t ShenandoahHeapRegion::get_shared_allocs() const { - return used() - (_tlab_allocs + _gclab_allocs) * HeapWordSize; + return used() - (_tlab_allocs + _gclab_allocs + _plab_allocs) * HeapWordSize; } size_t ShenandoahHeapRegion::get_tlab_allocs() const { @@ -332,6 +342,10 @@ size_t ShenandoahHeapRegion::get_gclab_allocs() const { return _gclab_allocs * HeapWordSize; } +size_t ShenandoahHeapRegion::get_plab_allocs() const { + return _plab_allocs * HeapWordSize; +} + void ShenandoahHeapRegion::set_live_data(size_t s) { assert(Thread::current()->is_VM_thread(), "by VM thread"); _live_data = (s >> LogHeapWordSize); @@ -397,6 +411,9 @@ void ShenandoahHeapRegion::print_on(outputStream* st) const { st->print("|U " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(used()), proper_unit_for_byte_size(used())); st->print("|T " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_tlab_allocs()), proper_unit_for_byte_size(get_tlab_allocs())); st->print("|G " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_gclab_allocs()), proper_unit_for_byte_size(get_gclab_allocs())); + if (ShenandoahHeap::heap()->mode()->is_generational()) { + st->print("|G " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_plab_allocs()), proper_unit_for_byte_size(get_plab_allocs())); + } st->print("|S " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_shared_allocs()), proper_unit_for_byte_size(get_shared_allocs())); st->print("|L " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_live_data_bytes()), proper_unit_for_byte_size(get_live_data_bytes())); st->print("|CP " SIZE_FORMAT_W(3), pin_count()); @@ -533,7 +550,6 @@ void ShenandoahHeapRegion::recycle() { set_update_watermark(bottom()); make_empty(); - set_affiliation(FREE); if (ZapUnusedHeapArea) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp index 4f77892fd8bb4..70a58a8f0e2a7 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp @@ -168,7 +168,7 @@ class ShenandoahHeapRegion { } // Allowed transitions from the outside code: - void make_regular_allocation(); + void make_regular_allocation(ShenandoahRegionAffiliation affiliation); void make_regular_bypass(); void make_humongous_start(); void make_humongous_cont(); @@ -242,6 +242,7 @@ class ShenandoahHeapRegion { size_t _tlab_allocs; size_t _gclab_allocs; + size_t _plab_allocs; volatile size_t _live_data; volatile size_t _critical_pins; @@ -339,7 +340,7 @@ class ShenandoahHeapRegion { } // Allocation (return NULL if full) - inline HeapWord* allocate(size_t word_size, ShenandoahAllocRequest::Type type); + inline HeapWord* allocate(size_t word_size, ShenandoahAllocRequest req); inline void clear_live_data(); void set_live_data(size_t s); @@ -391,6 +392,7 @@ class ShenandoahHeapRegion { size_t get_shared_allocs() const; size_t get_tlab_allocs() const; size_t get_gclab_allocs() const; + size_t get_plab_allocs() const; inline HeapWord* get_update_watermark() const; inline void set_update_watermark(HeapWord* w); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp index 461aa30fcbee1..55f26e93241a2 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp @@ -30,14 +30,14 @@ #include "gc/shenandoah/shenandoahPacer.inline.hpp" #include "runtime/atomic.hpp" -HeapWord* ShenandoahHeapRegion::allocate(size_t size, ShenandoahAllocRequest::Type type) { +HeapWord* ShenandoahHeapRegion::allocate(size_t size, ShenandoahAllocRequest req) { shenandoah_assert_heaplocked_or_safepoint(); assert(is_object_aligned(size), "alloc size breaks alignment: " SIZE_FORMAT, size); HeapWord* obj = top(); if (pointer_delta(end(), obj) >= size) { - make_regular_allocation(); - adjust_alloc_metadata(type, size); + make_regular_allocation(req.affiliation()); + adjust_alloc_metadata(req.type(), size); HeapWord* new_top = obj + size; set_top(new_top); @@ -63,6 +63,9 @@ inline void ShenandoahHeapRegion::adjust_alloc_metadata(ShenandoahAllocRequest:: case ShenandoahAllocRequest::_alloc_gclab: _gclab_allocs += size; break; + case ShenandoahAllocRequest::_alloc_plab: + _plab_allocs += size; + break; default: ShouldNotReachHere(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp index dfdd95d15799d..4b4771cc41f8e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp @@ -111,6 +111,7 @@ void ShenandoahHeapRegionCounters::update() { data |= ((100 * r->get_live_data_bytes() / rs) & PERCENT_MASK) << LIVE_SHIFT; data |= ((100 * r->get_tlab_allocs() / rs) & PERCENT_MASK) << TLAB_SHIFT; data |= ((100 * r->get_gclab_allocs() / rs) & PERCENT_MASK) << GCLAB_SHIFT; + data |= ((100 * r->get_plab_allocs() / rs) & PERCENT_MASK) << PLAB_SHIFT; data |= ((100 * r->get_shared_allocs() / rs) & PERCENT_MASK) << SHARED_SHIFT; data |= (r->age() & AGE_MASK) << AGE_SHIFT; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.hpp index 5400585434a1b..ebeebaa237ed6 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.hpp @@ -51,7 +51,7 @@ * - bits 14-20 tlab allocated memory in percent * - bits 21-27 gclab allocated memory in percent * - bits 28-34 shared allocated memory in percent - * - bits 35-41 + * - bits 35-41 plab allocated memory in percent * - bits 42-50 * - bits 51-55 age * - bits 56-57 affiliation: 0 = free, young = 1, old = 2 @@ -70,6 +70,7 @@ class ShenandoahHeapRegionCounters : public CHeapObj { static const jlong TLAB_SHIFT = 14; static const jlong GCLAB_SHIFT = 21; static const jlong SHARED_SHIFT = 28; + static const jlong PLAB_SHIFT = 35; static const jlong AGE_SHIFT = 51; static const jlong AFFILIATION_SHIFT = 56; static const jlong STATUS_SHIFT = 58; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp b/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp index e34073f9b8ee7..504cb84685f30 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp @@ -47,6 +47,8 @@ class ShenandoahThreadLocalData { SATBMarkQueue _satb_mark_queue; PLAB* _gclab; size_t _gclab_size; + PLAB* _plab; + size_t _plab_size; uint _worker_id; int _disarmed_value; double _paced_time; @@ -58,6 +60,8 @@ class ShenandoahThreadLocalData { _satb_mark_queue(&ShenandoahBarrierSet::satb_mark_queue_set()), _gclab(NULL), _gclab_size(0), + _plab(NULL), + _plab_size(0), _worker_id(INVALID_WORKER_ID), _disarmed_value(0), _paced_time(0) { @@ -71,6 +75,9 @@ class ShenandoahThreadLocalData { if (_gclab != NULL) { delete _gclab; } + if (_plab != NULL) { + delete _plab; + } } static ShenandoahThreadLocalData* data(Thread* thread) { @@ -118,6 +125,8 @@ class ShenandoahThreadLocalData { assert(data(thread)->_gclab == NULL, "Only initialize once"); data(thread)->_gclab = new PLAB(PLAB::min_size()); data(thread)->_gclab_size = 0; + data(thread)->_plab = new PLAB(PLAB::min_size()); + data(thread)->_plab_size = 0; } static PLAB* gclab(Thread* thread) { @@ -132,6 +141,18 @@ class ShenandoahThreadLocalData { data(thread)->_gclab_size = v; } + static PLAB* plab(Thread* thread) { + return data(thread)->_plab; + } + + static size_t plab_size(Thread* thread) { + return data(thread)->_plab_size; + } + + static void set_plab_size(Thread* thread, size_t v) { + data(thread)->_plab_size = v; + } + static void add_paced_time(Thread* thread, double v) { data(thread)->_paced_time += v; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp index 8423bd51934f9..d2742d584fb2f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp @@ -435,8 +435,11 @@ class ShenandoahVerifyHeapRegionClosure : public ShenandoahHeapRegionClosure { verify(r, r->get_gclab_allocs() <= r->capacity(), "GCLAB alloc count should not be larger than capacity"); - verify(r, r->get_shared_allocs() + r->get_tlab_allocs() + r->get_gclab_allocs() == r->used(), - "Accurate accounting: shared + TLAB + GCLAB = used"); + verify(r, r->get_plab_allocs() <= r->capacity(), + "PLAB alloc count should not be larger than capacity"); + + verify(r, r->get_shared_allocs() + r->get_tlab_allocs() + r->get_gclab_allocs() + r->get_plab_allocs() == r->used(), + "Accurate accounting: shared + TLAB + GCLAB + PLAB = used"); verify(r, !r->is_empty() || !r->has_live(), "Empty regions should not have live data"); diff --git a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp index 4babedb7cc2fc..e746dd2b42f17 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp @@ -224,6 +224,10 @@ product(bool, ShenandoahElasticTLAB, true, DIAGNOSTIC, \ "Use Elastic TLABs with Shenandoah") \ \ + product(bool, ShenandoahUsePLAB, true, DIAGNOSTIC, \ + "Use PLABs for object promotions with Shenandoah, " \ + "if in generational mode and UseTLAB is also set.") \ + \ product(uintx, ShenandoahEvacReserve, 5, EXPERIMENTAL, \ "How much of heap to reserve for evacuations. Larger values make "\ "GC evacuate more live objects on every cycle, while leaving " \ From cbc9e448f18848ab2211ea5d9a58146d5382d174 Mon Sep 17 00:00:00 2001 From: Aleksey Shipilev Date: Tue, 4 May 2021 11:27:55 +0000 Subject: [PATCH 038/254] Fix Windows builds Reviewed-by: rkennke --- .../shenandoah/heuristics/shenandoahOldHeuristics.cpp | 8 ++++---- src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp | 11 ++++++----- src/hotspot/share/gc/shenandoah/shenandoahOldGC.hpp | 2 ++ 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp index e6ba41679d925..d55737917d34b 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp @@ -236,9 +236,9 @@ void ShenandoahOldHeuristics::prepare_for_old_collections() { if (percent_garbage < collection_threshold_garbage_percent) { _hidden_next_old_collection_candidate = 0; - _hidden_old_collection_candidates = i; - _first_coalesce_and_fill_candidate = i; - _old_coalesce_and_fill_candidates = cand_idx - i; + _hidden_old_collection_candidates = (uint)i; + _first_coalesce_and_fill_candidate = (uint)i; + _old_coalesce_and_fill_candidates = (uint)(cand_idx - i); // Note that we do not coalesce and fill occupied humongous regions // HR: humongous regions, RR: regular regions, CF: coalesce and fill regions @@ -252,7 +252,7 @@ void ShenandoahOldHeuristics::prepare_for_old_collections() { // If we reach here, all of non-humogous old-gen regions are candidates for collection set. _hidden_next_old_collection_candidate = 0; - _hidden_old_collection_candidates = first_humongous_non_empty; + _hidden_old_collection_candidates = (uint)first_humongous_non_empty; _first_coalesce_and_fill_candidate = 0; _old_coalesce_and_fill_candidates = 0; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp index 4b81b557ee26a..08586e18ce9ee 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp @@ -67,7 +67,9 @@ class ShenandoahConcurrentCoalesceAndFillTask : public AbstractGangTask { ShenandoahOldGC::ShenandoahOldGC(ShenandoahGeneration* generation, ShenandoahSharedFlag& allow_preemption) : - ShenandoahConcurrentGC(generation), _allow_preemption(allow_preemption) {} + ShenandoahConcurrentGC(generation), _allow_preemption(allow_preemption) { + _coalesce_and_fill_region_array = NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, ShenandoahHeap::heap()->num_regions(), mtGC); +} void ShenandoahOldGC::entry_old_evacuations() { ShenandoahHeap* heap = ShenandoahHeap::heap(); @@ -146,10 +148,9 @@ void ShenandoahOldGC::op_coalesce_and_fill() { ShenandoahOldHeuristics* old_heuristics = heap->old_heuristics(); uint coalesce_and_fill_regions_count = old_heuristics->old_coalesce_and_fill_candidates(); - ShenandoahHeapRegion* coalesce_and_fill_region_array[coalesce_and_fill_regions_count]; - - old_heuristics->get_coalesce_and_fill_candidates(coalesce_and_fill_region_array); - ShenandoahConcurrentCoalesceAndFillTask task(nworkers, coalesce_and_fill_region_array, coalesce_and_fill_regions_count); + assert(coalesce_and_fill_regions_count <= heap->num_regions(), "Sanity"); + old_heuristics->get_coalesce_and_fill_candidates(_coalesce_and_fill_region_array); + ShenandoahConcurrentCoalesceAndFillTask task(nworkers, _coalesce_and_fill_region_array, coalesce_and_fill_regions_count); // TODO: We need to implement preemption of coalesce and fill. If young-gen wants to run while we're working on this, diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.hpp index a613c20920c3e..55323f283311a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.hpp @@ -35,6 +35,8 @@ class ShenandoahOldGC : public ShenandoahConcurrentGC { ShenandoahOldGC(ShenandoahGeneration* generation, ShenandoahSharedFlag& allow_preemption); bool collect(GCCause::Cause cause); private: + ShenandoahHeapRegion** _coalesce_and_fill_region_array; + void entry_old_evacuations(); void entry_coalesce_and_fill(); ShenandoahSharedFlag& _allow_preemption; From 41ed8466724ff576815ac35a1f89ca55f105eca6 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Thu, 6 May 2021 15:38:11 +0000 Subject: [PATCH 039/254] Fix Region Promotion Co-authored-by: Bernd Mathiske Reviewed-by: rkennke --- .../share/gc/shenandoah/shenandoahHeap.cpp | 68 +++++++++---------- .../share/gc/shenandoah/shenandoahHeap.hpp | 2 + .../gc/shenandoah/shenandoahHeapRegion.cpp | 38 +++++------ .../gc/shenandoah/shenandoahHeapRegion.hpp | 6 ++ .../shenandoahHeapRegion.inline.hpp | 12 ++++ .../shenandoah/shenandoahThreadLocalData.hpp | 9 +++ .../shenandoah/shenandoahYoungGeneration.cpp | 14 ++-- 7 files changed, 82 insertions(+), 67 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index b757379be4d68..bd07622eac2b6 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -1012,43 +1012,6 @@ HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) { HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req, bool& in_new_region) { ShenandoahHeapLocker locker(lock()); HeapWord* result = _free_set->allocate(req, in_new_region); - // Register the newly allocated object while we're holding the global lock since there's no synchronization - // built in to the implementation of register_object(). There are potential races when multiple independent - // threads are allocating objects, some of which might span the same card region. For example, consider - // a card table's memory region within which three objects are being allocated by three different threads: - // - // objects being "concurrently" allocated: - // [-----a------][-----b-----][--------------c------------------] - // [---- card table memory range --------------] - // - // Before any objects are allocated, this card's memory range holds no objects. Note that: - // allocation of object a wants to set the has-object, first-start, and last-start attributes of the preceding card region. - // allocation of object b wants to set the has-object, first-start, and last-start attributes of this card region. - // allocation of object c also wants to set the has-object, first-start, and last-start attributes of this card region. - // - // The thread allocating b and the thread allocating c can "race" in various ways, resulting in confusion, such as last-start - // representing object b while first-start represents object c. This is why we need to require all register_object() - // invocations to be "mutually exclusive". Later, when we use GCLABs and PLABs to allocate memory for promotions and evacuations, - // the protocol may work something like the following: - // 1. The GCLAB/PLAB is allocated by this (or similar) function, while holding the global lock. - // 2. The GCLAB/PLAB is always aligned at the start of a card memory range - // and is always a multiple of the card-table memory range size. - // 3. Individual allocations carved from a GCLAB/PLAB are not immediately registered. - // 4. A PLAB is registered as a single object. - // 5. When a PLAB is eventually retired, all of the objects allocated within the GCLAB/PLAB are registered in batch by a - // single thread. No further synchronization is required because no other allocations will pertain to the same - // card-table memory ranges. - // - // The other case that needs special handling is region promotion. When a region is promoted, all objects contained - // in it are registered. Since the region is a multiple of card table memory range sizes, there is no need for - // synchronization. - // TODO: figure out how to allow multiple threads to work together to register all of the objects in - // a promoted region, or at least try to balance the efforts so that different GC threads work - // on registering the objects of different heap regions. - // - if (mode()->is_generational() && result != NULL && req.affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) { - ShenandoahHeap::heap()->card_scan()->register_object(result); - } return result; } @@ -1299,6 +1262,37 @@ void ShenandoahHeap::gclabs_retire(bool resize) { } } +class ShenandoahTagGCLABClosure : public ThreadClosure { +public: + void do_thread(Thread* thread) { + PLAB* gclab = ShenandoahThreadLocalData::gclab(thread); + assert(gclab != NULL, "GCLAB should be initialized for %s", thread->name()); + if (gclab->words_remaining() > 0) { + ShenandoahHeapRegion* r = ShenandoahHeap::heap()->heap_region_containing(gclab->allocate(0)); + r->set_young_lab_flag(); + } + } +}; + +void ShenandoahHeap::set_young_lab_region_flags() { + if (!UseTLAB) { + return; + } + for (size_t i = 0; i < _num_regions; i++) { + _regions[i]->clear_young_lab_flags(); + } + ShenandoahTagGCLABClosure cl; + workers()->threads_do(&cl); + for (JavaThreadIteratorWithHandle jtiwh; JavaThread *t = jtiwh.next(); ) { + cl.do_thread(t); + ThreadLocalAllocBuffer& tlab = t->tlab(); + if (tlab.end() != NULL) { + ShenandoahHeapRegion* r = heap_region_containing(tlab.start()); + r->set_young_lab_flag(); + } + } +} + // Returns size in bytes size_t ShenandoahHeap::unsafe_max_tlab_alloc(Thread *thread) const { if (ShenandoahElasticTLAB) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index c648fdb517c47..0a36a5a9c8637 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -598,6 +598,8 @@ class ShenandoahHeap : public CollectedHeap { void tlabs_retire(bool resize); void gclabs_retire(bool resize); + void set_young_lab_region_flags(); + // ---------- Marking support // private: diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index 338dee4d72fdf..c98a27881f2f7 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -71,6 +71,7 @@ ShenandoahHeapRegion::ShenandoahHeapRegion(HeapWord* start, size_t index, bool c _tlab_allocs(0), _gclab_allocs(0), _plab_allocs(0), + _has_young_lab(false), _live_data(0), _critical_pins(0), _update_watermark(start), @@ -867,44 +868,37 @@ size_t ShenandoahHeapRegion::promote() { if (is_humongous_start()) { oop obj = oop(bottom()); assert(marking_context->is_marked(obj), "promoted humongous object should be alive"); - - size_t index_limit = index() + ShenandoahHeapRegion::required_regions(obj->size() * HeapWordSize); heap->card_scan()->register_object(bottom()); + size_t index_limit = index() + ShenandoahHeapRegion::required_regions(obj->size() * HeapWordSize); for (size_t i = index(); i < index_limit; i++) { ShenandoahHeapRegion* r = heap->get_region(i); - log_debug(gc)("promoting region " SIZE_FORMAT ", clear cards from " SIZE_FORMAT " to " SIZE_FORMAT, + log_debug(gc)("promoting region " SIZE_FORMAT ", from " SIZE_FORMAT " to " SIZE_FORMAT, r->index(), (size_t) r->bottom(), (size_t) r->top()); - - ShenandoahBarrierSet::barrier_set()->card_table()->clear_MemRegion(MemRegion(r->bottom(), r->end())); + if (top() < end()) { + ShenandoahHeap::fill_with_object(top(), (end() - top()) / HeapWordSize); + heap->card_scan()->register_object(top()); + ShenandoahBarrierSet::barrier_set()->card_table()->clear_MemRegion(MemRegion(top(), end())); + } + ShenandoahBarrierSet::barrier_set()->card_table()->dirty_MemRegion(MemRegion(bottom(), top())); r->set_affiliation(OLD_GENERATION); old_generation->increase_used(r->used()); young_generation->decrease_used(r->used()); } - // HEY! Better to call ShenandoahHeap::heap()->card_scan()->mark_range_as_clean(r->bottom(), obj->size()) - // and skip the calls to clear_MemRegion() above. - - // Iterate over all humongous regions that are spanned by the humongous object obj. The remnant - // of memory in the last humongous region that is not spanned by obj is currently not used. - obj->oop_iterate(&update_card_values); return index_limit - index(); } else { - log_debug(gc)("promoting region " SIZE_FORMAT ", clear cards from " SIZE_FORMAT " to " SIZE_FORMAT, + log_debug(gc)("promoting region " SIZE_FORMAT ", from " SIZE_FORMAT " to " SIZE_FORMAT, index(), (size_t) bottom(), (size_t) top()); assert(!is_humongous_continuation(), "should not promote humongous object continuation in isolation"); - // Rather than scanning entire contents of the promoted region right now to determine which - // cards to mark dirty, we just mark them all as dirty. Later, when we scan the remembered - // set, we will clear cards that are found to not contain live references to young memory. - // Ultimately, this approach is more efficient as it only scans the "dirty" cards once and - // the clean cards once. The alternative approach of scanning all cards now and then scanning - // dirty cards again at next concurrent mark pass scans the clean cards once and the dirty - // cards twice. - - // HEY! Better to call ShenandoahHeap::heap()->card_scan()->mark_range_as_dirty(r->bottom(), obj->size()); - ShenandoahBarrierSet::barrier_set()->card_table()->dirty_MemRegion(MemRegion(bottom(), end())); set_affiliation(OLD_GENERATION); old_generation->increase_used(used()); young_generation->decrease_used(used()); + + ShenandoahBarrierSet::barrier_set()->card_table()->clear_MemRegion(MemRegion(top(), end())); + // In terms of card marking, We could just set the whole occupied range in this region to dirty instead of iterating here. + // Card scanning could correct false positives later and that would be more efficient. + // But oop_iterate_objects() has other, indispensable effects: filling dead objects and registering object starts. + // So while we are already doing this here, we may as well also set more precise card values. oop_iterate_objects(&update_card_values, /*fill_dead_objects*/ true, /* reregister_coalesced_objects */ false); return 1; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp index 70a58a8f0e2a7..7ae228de8206a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp @@ -212,6 +212,10 @@ class ShenandoahHeapRegion { void record_unpin(); size_t pin_count() const; + void clear_young_lab_flags(); + void set_young_lab_flag(); + bool has_young_lab_flag(); + private: static size_t RegionCount; static size_t RegionSizeBytes; @@ -244,6 +248,8 @@ class ShenandoahHeapRegion { size_t _gclab_allocs; size_t _plab_allocs; + bool _has_young_lab; + volatile size_t _live_data; volatile size_t _critical_pins; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp index 55f26e93241a2..316a1c0277e34 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp @@ -135,4 +135,16 @@ inline void ShenandoahHeapRegion::set_update_watermark_at_safepoint(HeapWord* w) _update_watermark = w; } +inline void ShenandoahHeapRegion::clear_young_lab_flags() { + _has_young_lab = false; +} + +inline void ShenandoahHeapRegion::set_young_lab_flag() { + _has_young_lab = true; +} + +inline bool ShenandoahHeapRegion::has_young_lab_flag() { + return _has_young_lab; +} + #endif // SHARE_GC_SHENANDOAH_SHENANDOAHHEAPREGION_INLINE_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp b/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp index 504cb84685f30..cebdaebc99bde 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp @@ -45,10 +45,19 @@ class ShenandoahThreadLocalData { uint8_t _oom_scope_nesting_level; bool _oom_during_evac; SATBMarkQueue _satb_mark_queue; + + // Thread-local allocation buffer for object evacuations. + // In generational mode, it is exclusive to the young generation. PLAB* _gclab; size_t _gclab_size; + + // Thread-local allocation buffer only used in generational mode. + // Used both by mutator threads and by GC worker threads + // for evacuations within the old generation and + // for promotions from the young generation into the old generation. PLAB* _plab; size_t _plab_size; + uint _worker_id; int _disarmed_value; double _paced_time; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp index 653d011136053..b724ee808d121 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp @@ -68,14 +68,11 @@ class ShenandoahPromoteTenuredRegionsTask : public AbstractGangTask { ShenandoahParallelWorkerSession worker_session(worker_id); ShenandoahHeapRegion* r = _regions->next(); while (r != NULL) { - if (r->is_young()) { - // The thread that first encounters a humongous start region is responsible - // for promoting the continuation regions so we need this guard here to - // keep other worker threads from trying to promote the continuations. - if (r->age() >= InitialTenuringThreshold && !r->is_humongous_continuation()) { - size_t promoted = r->promote(); - Atomic::add(&_promoted, promoted); - } + if (r->is_young() && r->age() >= InitialTenuringThreshold && ((r->is_regular() && !r->has_young_lab_flag()) || r->is_humongous_start())) { + // The above condition filtered out humongous continuations, among other states. + // Here we rely on promote() below promoting related continuation regions when encountering a homongous start. + size_t promoted = r->promote(); + Atomic::add(&_promoted, promoted); } r = _regions->next(); } @@ -83,6 +80,7 @@ class ShenandoahPromoteTenuredRegionsTask : public AbstractGangTask { }; void ShenandoahYoungGeneration::promote_tenured_regions() { + ShenandoahHeap::heap()->set_young_lab_region_flags(); ShenandoahRegionIterator regions; ShenandoahPromoteTenuredRegionsTask task(®ions); ShenandoahHeap::heap()->workers()->run_task(&task); From c6740a8fa30be0e0977ae69ea255c76fc306491c Mon Sep 17 00:00:00 2001 From: William Kemper Date: Fri, 14 May 2021 16:13:44 +0000 Subject: [PATCH 040/254] More promotion fixes Reviewed-by: rkennke --- .../share/gc/shenandoah/shenandoahHeap.hpp | 4 +++- .../gc/shenandoah/shenandoahHeap.inline.hpp | 18 ++++++++++++++--- .../gc/shenandoah/shenandoahHeapRegion.cpp | 20 +++++++++---------- .../gc/shenandoah/shenandoahMark.inline.hpp | 2 +- 4 files changed, 29 insertions(+), 15 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index 0a36a5a9c8637..3f4eeda47215a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -683,7 +683,9 @@ class ShenandoahHeap : public CollectedHeap { public: inline RememberedScanner* card_scan() { return _card_scan; } void clear_cards_for(ShenandoahHeapRegion* region); - void mark_card_as_dirty(HeapWord* location); + void dirty_cards(HeapWord* start, HeapWord* end); + void clear_cards(HeapWord* start, HeapWord* end); + void mark_card_as_dirty(void* location); void retire_plab(PLAB* plab); // ---------- Helper functions diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp index 19231813d43ae..fa0bae48fefe5 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp @@ -626,13 +626,25 @@ inline ShenandoahMarkingContext* ShenandoahHeap::marking_context() const { inline void ShenandoahHeap::clear_cards_for(ShenandoahHeapRegion* region) { if (mode()->is_generational()) { - _card_scan->mark_range_as_empty(region->bottom(), (uint32_t) (region->end() - region->bottom())); + _card_scan->mark_range_as_empty(region->bottom(), pointer_delta(region->end(), region->bottom())); } } -inline void ShenandoahHeap::mark_card_as_dirty(HeapWord* location) { +inline void ShenandoahHeap::dirty_cards(HeapWord* start, HeapWord* end) { + assert(mode()->is_generational(), "Should only be used for generational mode"); + size_t words = pointer_delta(end, start); + _card_scan->mark_range_as_dirty(start, words); +} + +inline void ShenandoahHeap::clear_cards(HeapWord* start, HeapWord* end) { + assert(mode()->is_generational(), "Should only be used for generational mode"); + size_t words = pointer_delta(end, start); + _card_scan->mark_range_as_clean(start, words); +} + +inline void ShenandoahHeap::mark_card_as_dirty(void* location) { if (mode()->is_generational()) { - _card_scan->mark_card_as_dirty(location); + _card_scan->mark_card_as_dirty((HeapWord*)location); } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index c98a27881f2f7..b2a5f05cc11be 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -483,6 +483,7 @@ void ShenandoahHeapRegion::oop_iterate_objects(OopIterateClosure* blk, bool fill ShenandoahHeap* heap = ShenandoahHeap::heap(); ShenandoahMarkingContext* marking_context = heap->marking_context(); assert(heap->active_generation()->is_mark_complete(), "sanity"); + HeapWord* tams = marking_context->top_at_mark_start(this); while (obj_addr < t) { oop obj = oop(obj_addr); @@ -494,8 +495,8 @@ void ShenandoahHeapRegion::oop_iterate_objects(OopIterateClosure* blk, bool fill obj_addr += obj->oop_iterate_size(blk); } else { // Object is not marked. Coalesce and fill dead object with dead neighbors. - HeapWord* next_marked_obj = marking_context->get_next_marked_addr(obj_addr, t); - assert(next_marked_obj <= t, "next marked object cannot exceed top"); + HeapWord* next_marked_obj = marking_context->get_next_marked_addr(obj_addr, tams); + assert(next_marked_obj <= tams, "next marked object cannot exceed top at mark start"); size_t fill_size = next_marked_obj - obj_addr; ShenandoahHeap::fill_with_object(obj_addr, fill_size); if (reregister_coalesced_objects) { @@ -830,8 +831,7 @@ class UpdateCardValuesClosure : public BasicOopIterateClosure { private: void update_card_value(void* address, oop obj) { if (ShenandoahHeap::heap()->is_in_young(obj)) { - volatile CardTable::CardValue* card_value = ShenandoahBarrierSet::barrier_set()->card_table()->byte_for(address); - *card_value = CardTable::dirty_card_val(); + ShenandoahHeap::heap()->mark_card_as_dirty(address); } } @@ -874,12 +874,12 @@ size_t ShenandoahHeapRegion::promote() { ShenandoahHeapRegion* r = heap->get_region(i); log_debug(gc)("promoting region " SIZE_FORMAT ", from " SIZE_FORMAT " to " SIZE_FORMAT, r->index(), (size_t) r->bottom(), (size_t) r->top()); - if (top() < end()) { - ShenandoahHeap::fill_with_object(top(), (end() - top()) / HeapWordSize); - heap->card_scan()->register_object(top()); - ShenandoahBarrierSet::barrier_set()->card_table()->clear_MemRegion(MemRegion(top(), end())); + if (r->top() < r->end()) { + ShenandoahHeap::fill_with_object(r->top(), (r->end() - r->top()) / HeapWordSize); + heap->card_scan()->register_object(r->top()); + heap->clear_cards(r->top(), r->end()); } - ShenandoahBarrierSet::barrier_set()->card_table()->dirty_MemRegion(MemRegion(bottom(), top())); + heap->dirty_cards(r->bottom(), r->top()); r->set_affiliation(OLD_GENERATION); old_generation->increase_used(r->used()); young_generation->decrease_used(r->used()); @@ -894,11 +894,11 @@ size_t ShenandoahHeapRegion::promote() { old_generation->increase_used(used()); young_generation->decrease_used(used()); - ShenandoahBarrierSet::barrier_set()->card_table()->clear_MemRegion(MemRegion(top(), end())); // In terms of card marking, We could just set the whole occupied range in this region to dirty instead of iterating here. // Card scanning could correct false positives later and that would be more efficient. // But oop_iterate_objects() has other, indispensable effects: filling dead objects and registering object starts. // So while we are already doing this here, we may as well also set more precise card values. + heap->dirty_cards(bottom(), top()); oop_iterate_objects(&update_card_values, /*fill_dead_objects*/ true, /* reregister_coalesced_objects */ false); return 1; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp index 5f9205d970927..125df2774e9b3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp @@ -275,7 +275,7 @@ inline void ShenandoahMark::mark_through_ref(T *p, ShenandoahObjToScanQueue* q, } else if (GENERATION == OLD) { // Old mark, found a young pointer. assert(ShenandoahHeap::heap()->is_in_young(obj), "Expected young object."); - ShenandoahHeap::heap()->mark_card_as_dirty((HeapWord*)p); + ShenandoahHeap::heap()->mark_card_as_dirty(p); } } } From 207da43eeaf7dc7817fae622a8c36c83fe3e1d1e Mon Sep 17 00:00:00 2001 From: William Kemper Date: Fri, 14 May 2021 16:14:43 +0000 Subject: [PATCH 041/254] Abandon old gen cset candidates upon global collection Reviewed-by: rkennke --- .../gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp | 9 +++++++++ .../gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp | 4 ++++ src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp | 2 +- .../share/gc/shenandoah/shenandoahGlobalGeneration.cpp | 9 +++++++++ .../share/gc/shenandoah/shenandoahGlobalGeneration.hpp | 2 ++ src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp | 5 +++++ src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp | 1 + 7 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp index d55737917d34b..0747121a57a3b 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp @@ -315,3 +315,12 @@ bool ShenandoahOldHeuristics::should_defer_gc() { return false; } +void ShenandoahOldHeuristics::abandon_collection_candidates() { + _old_collection_candidates = 0; + _next_old_collection_candidate = 0; + _hidden_old_collection_candidates = 0; + _hidden_next_old_collection_candidate = 0; + _old_coalesce_and_fill_candidates = 0; + _first_coalesce_and_fill_candidate = 0; +} + diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp index f78fe89ff0f83..26f975305d480 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp @@ -100,6 +100,10 @@ class ShenandoahOldHeuristics : public ShenandoahHeuristics { bool should_defer_gc(); + // If a GLOBAL gc occurs, it will collect the entire heap which invalidates any collection candidates being + // held by this heuristic for supplying mixed collections. + void abandon_collection_candidates(); + }; #endif // SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHOLDHEURISTICS_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index 56591106d5e63..4e7a0a3b27c95 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -82,7 +82,7 @@ class ShenandoahGeneration : public CHeapObj { void reset_mark_bitmap(); // Used by concurrent and degenerated GC to reset regions. - void prepare_gc(); + virtual void prepare_gc(); void prepare_regions_and_collection_set(bool concurrent); // Cancel marking (used by Full collect and when cancelling cycle). diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp index 0533275c48d95..2ec2b36cc9141 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp @@ -87,3 +87,12 @@ bool ShenandoahGlobalGeneration::is_concurrent_mark_in_progress() { return heap->is_concurrent_mark_in_progress(); } +void ShenandoahGlobalGeneration::prepare_gc() { + ShenandoahGeneration::prepare_gc(); + + ShenandoahHeap* heap = ShenandoahHeap::heap(); + if (heap->mode()->is_generational()) { + heap->cancel_mixed_collections(); + } +} + diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp index 8cc55a8184521..30366e3236e62 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp @@ -42,6 +42,8 @@ class ShenandoahGlobalGeneration : public ShenandoahGeneration { virtual size_t used() const; virtual size_t available() const; + virtual void prepare_gc(); + virtual void set_concurrent_mark_in_progress(bool in_progress); bool contains(ShenandoahHeapRegion* region) const; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index bd07622eac2b6..60b03580cda5d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -894,6 +894,11 @@ void ShenandoahHeap::retire_plab(PLAB* plab) { } } +void ShenandoahHeap::cancel_mixed_collections() { + assert(_old_generation != NULL, "Should only have mixed collections in generation mode."); + _old_heuristics->abandon_collection_candidates(); +} + HeapWord* ShenandoahHeap::allocate_new_tlab(size_t min_size, size_t requested_size, size_t* actual_size) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index 3f4eeda47215a..07705302ecefa 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -687,6 +687,7 @@ class ShenandoahHeap : public CollectedHeap { void clear_cards(HeapWord* start, HeapWord* end); void mark_card_as_dirty(void* location); void retire_plab(PLAB* plab); + void cancel_mixed_collections(); // ---------- Helper functions // From 627ca1006a7239a87c327dee7a4ea342236ec725 Mon Sep 17 00:00:00 2001 From: Roman Kennke Date: Tue, 25 May 2021 12:56:08 +0200 Subject: [PATCH 042/254] Allow committers to review Shenandoah changes --- .jcheck/conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.jcheck/conf b/.jcheck/conf index 38aaf8e6387c9..bc49e711d6448 100644 --- a/.jcheck/conf +++ b/.jcheck/conf @@ -18,7 +18,7 @@ files=.*\.cpp|.*\.hpp|.*\.c|.*\.h|.*\.java|.*\.cc|.*\.hh|.*\.m|.*\.mm|.*\.gmk|.* ignore-tabs=.*\.gmk|Makefile [checks "reviewers"] -reviewers=1 +committers=1 ignore=duke [checks "committer"] From 27732e58e555860ea08eefc6980b780b4628be89 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Tue, 25 May 2021 17:21:05 +0000 Subject: [PATCH 043/254] Collect old humongous regions immediately at end of concurrent old mark Reviewed-by: rkennke --- .../heuristics/shenandoahOldHeuristics.cpp | 72 ++++++------------- .../share/gc/shenandoah/shenandoahHeap.cpp | 3 +- .../share/gc/shenandoah/shenandoahHeap.hpp | 2 +- 3 files changed, 23 insertions(+), 54 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp index 0747121a57a3b..1fd737d1061f3 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp @@ -165,59 +165,29 @@ void ShenandoahOldHeuristics::prepare_for_old_collections() { ShenandoahHeapRegion* region = heap->get_region(i); if (!in_generation(region)) { continue; - } else { - size_t garbage = region->garbage(); - total_garbage += garbage; + } + + size_t garbage = region->garbage(); + total_garbage += garbage; + if (region->is_regular()) { candidates[cand_idx]._region = region; candidates[cand_idx]._garbage = garbage; cand_idx++; - } - } - - // Give special treatment to humongous regions. Assume humongous regions is entirely - // garbage or entirely non-garbage. Assume that a head humongous region and the associated - // humongous continuous regions are uniformly entirely garbage or entirely non-garbage. - // - // Sift garbage humongous regions to front, non-garbage humongous regions to end of array. - size_t first_non_humongous_empty = 0; - size_t first_humongous_non_empty = cand_idx; - - size_t i = 0; - while (i < first_humongous_non_empty) { - ShenandoahHeapRegion* region = candidates[i]._region; - if (region->is_humongous()) { - if (region->get_live_data_bytes() == 0) { - // Humongous region is entirely garbage. Reclaim it. - if (i == first_non_humongous_empty) { - first_non_humongous_empty++; - } else { - RegionData swap_tmp = candidates[i]; - candidates[i] = candidates[first_non_humongous_empty]; - candidates[first_non_humongous_empty++] = swap_tmp; - } - i++; - } else { - // Humongous region is non garbage. Don't reclaim it. - if (i + 1 == first_humongous_non_empty) { - first_humongous_non_empty--; - i++; - } else { - RegionData swap_tmp = candidates[i]; - candidates[i] = candidates[--first_humongous_non_empty]; - candidates[first_humongous_non_empty] = swap_tmp; - // Do not increment i so we can revisit swapped entry on next iteration - } + } else if (region->is_humongous_start()) { + if (!region->has_live()) { + // The humongous object is dead, we can just return this region and the continuations + // immediately to the freeset - no evacuations are necessary here. The continuations + // will be made into trash by this method, so they'll be skipped by the 'is_regular' + // check above. + size_t region_count = heap->trash_humongous_region_at(region); + log_debug(gc)("Trashed " SIZE_FORMAT " regions for humongous object.", region_count); } - } else { - i++; } } - // Prioritize regions to select garbage-first regions - QuickSort::sort(candidates + first_non_humongous_empty, (int)(first_humongous_non_empty - first_non_humongous_empty), - compare_by_garbage, false); + QuickSort::sort(candidates, cand_idx, compare_by_garbage, false); // Any old-gen region that contains (ShenandoahGarbageThreshold (default value 25))% garbage or more is to // be evacuated. @@ -230,7 +200,7 @@ void ShenandoahOldHeuristics::prepare_for_old_collections() { const size_t collection_threshold_garbage_percent = ShenandoahGarbageThreshold; size_t region_size = ShenandoahHeapRegion::region_size_bytes(); - for (size_t i = first_non_humongous_empty; i < first_humongous_non_empty; i++) { + for (size_t i = 0; i < cand_idx; i++) { // Do approximate percent to avoid floating point math size_t percent_garbage = candidates[i]._garbage * 100 / region_size; @@ -242,9 +212,8 @@ void ShenandoahOldHeuristics::prepare_for_old_collections() { // Note that we do not coalesce and fill occupied humongous regions // HR: humongous regions, RR: regular regions, CF: coalesce and fill regions - log_info(gc)("Old-gen mark evac (%u HR, %llu RR), %llu CF)", - (unsigned int) first_non_humongous_empty, - (unsigned long long) (_hidden_old_collection_candidates - first_non_humongous_empty), + log_info(gc)("Old-gen mark evac (%llu RR), %llu CF)", + (unsigned long long) (_hidden_old_collection_candidates), (unsigned long long) _old_coalesce_and_fill_candidates); return; } @@ -252,15 +221,14 @@ void ShenandoahOldHeuristics::prepare_for_old_collections() { // If we reach here, all of non-humogous old-gen regions are candidates for collection set. _hidden_next_old_collection_candidate = 0; - _hidden_old_collection_candidates = (uint)first_humongous_non_empty; + _hidden_old_collection_candidates = (uint)cand_idx; _first_coalesce_and_fill_candidate = 0; _old_coalesce_and_fill_candidates = 0; // Note that we do not coalesce and fill occupied humongous regions // HR: humongous regions, RR: regular regions, CF: coalesce and fill regions - log_info(gc)("Old-gen mark evac (%u HR, %llu RR), %llu CF)", - (unsigned int) first_non_humongous_empty, - (unsigned long long) (_hidden_old_collection_candidates - first_non_humongous_empty), + log_info(gc)("Old-gen mark evac (%llu RR), %llu CF)", + (unsigned long long) (_hidden_old_collection_candidates), (unsigned long long) _old_coalesce_and_fill_candidates); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 60b03580cda5d..cb5fe77ed008c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -1155,7 +1155,7 @@ void ShenandoahHeap::print_heap_regions_on(outputStream* st) const { } } -void ShenandoahHeap::trash_humongous_region_at(ShenandoahHeapRegion* start) { +size_t ShenandoahHeap::trash_humongous_region_at(ShenandoahHeapRegion* start) { assert(start->is_humongous_start(), "reclaim regions starting with the first one"); oop humongous_obj = oop(start->bottom()); @@ -1175,6 +1175,7 @@ void ShenandoahHeap::trash_humongous_region_at(ShenandoahHeapRegion* start) { region->make_trash_immediate(); } + return required_regions; } class ShenandoahCheckCleanGCLABClosure : public ThreadClosure { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index 07705302ecefa..8cfb99c561245 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -702,7 +702,7 @@ class ShenandoahHeap : public CollectedHeap { static inline oop cas_oop(oop n, oop* addr, oop c); static inline oop cas_oop(oop n, narrowOop* addr, narrowOop c); - void trash_humongous_region_at(ShenandoahHeapRegion *r); + size_t trash_humongous_region_at(ShenandoahHeapRegion *r); void deduplicate_string(oop str); From a21de22639315fe7d5eb06ef9707e594046bd38b Mon Sep 17 00:00:00 2001 From: William Kemper Date: Tue, 25 May 2021 17:33:31 +0000 Subject: [PATCH 044/254] Mixed evacuation fixes Reviewed-by: rkennke --- .../gc/shenandoah/shenandoahCollectionSet.cpp | 4 ++ .../gc/shenandoah/shenandoahCollectionSet.hpp | 7 ++- .../share/gc/shenandoah/shenandoahHeap.cpp | 45 ++++++++++--------- .../gc/shenandoah/shenandoahHeap.inline.hpp | 4 -- .../gc/shenandoah/shenandoahOldGeneration.cpp | 10 +++-- .../shenandoahScanRemembered.inline.hpp | 2 +- 6 files changed, 41 insertions(+), 31 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp index 385408d7fd562..da2658334dd47 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp @@ -40,6 +40,7 @@ ShenandoahCollectionSet::ShenandoahCollectionSet(ShenandoahHeap* heap, ReservedS _cset_map(_map_space.base() + ((uintx)heap_base >> _region_size_bytes_shift)), _biased_cset_map(_map_space.base()), _heap(heap), + _has_old_regions(false), _garbage(0), _used(0), _region_count(0), @@ -87,6 +88,7 @@ void ShenandoahCollectionSet::add_region(ShenandoahHeapRegion* r) { _region_count++; _garbage += r->garbage(); _used += r->used(); + _has_old_regions |= r->is_old(); // Update the region status too. State transition would be checked internally. r->make_cset(); @@ -107,6 +109,8 @@ void ShenandoahCollectionSet::clear() { _region_count = 0; _current_index = 0; + + _has_old_regions = false; } ShenandoahHeapRegion* ShenandoahCollectionSet::claim_next() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp index 56e96522094fd..39a34d6cb1db6 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp @@ -42,6 +42,7 @@ class ShenandoahCollectionSet : public CHeapObj { ShenandoahHeap* const _heap; + bool _has_old_regions; size_t _garbage; size_t _used; size_t _region_count; @@ -76,8 +77,10 @@ class ShenandoahCollectionSet : public CHeapObj { void print_on(outputStream* out) const; - size_t used() const { return _used; } - size_t garbage() const { return _garbage; } + bool has_old_regions() const { return _has_old_regions; } + size_t used() const { return _used; } + + size_t garbage() const { return _garbage; } void clear(); private: diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index cb5fe77ed008c..203385df1f828 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -2131,7 +2131,7 @@ class ShenandoahUpdateHeapRefsTask : public AbstractGangTask { // We update references for global, old, and young collections. assert(_heap->active_generation()->is_mark_complete(), "Expected complete marking"); ShenandoahMarkingContext* const ctx = _heap->marking_context(); - + bool is_mixed = _heap->collection_set()->has_old_regions(); while (r != NULL) { HeapWord* update_watermark = r->get_update_watermark(); assert (update_watermark >= r->bottom(), "sanity"); @@ -2141,31 +2141,17 @@ class ShenandoahUpdateHeapRefsTask : public AbstractGangTask { _heap->marked_object_oop_iterate(r, &cl, update_watermark); } else { assert(r->affiliation() == OLD_GENERATION, "Should not be updating references on FREE regions"); - if (!_heap->is_gc_generation_young()) { - // Old region in an old or global cycle. - // We need to make sure that the next cycle does not iterate over dead objects + if (!_heap->is_gc_generation_young() || is_mixed) { + // Old region in global or mixed cycle (in which case, old regions should be marked). + // We need to make sure that the next remembered set scan does not iterate over dead objects // which haven't had their references updated. r->oop_iterate(&cl, /*fill_dead_objects*/ true, /* reregister_coalesced_objects */ true); } else { - // Old region in a young cycle. + // Old region in a young cycle with no old regions. if (!ShenandoahUseSimpleCardScanning) { _heap->card_scan()->process_region(r, &cl); } else if (ShenandoahBarrierSet::barrier_set()->card_table()->is_dirty(MemRegion(r->bottom(), r->top()))) { - if (r->is_humongous()) { - r->oop_iterate_humongous(&cl); - } else { - // We don't have liveness information about this region. - // Therefore we process all objects, rather than just marked ones. - // Otherwise subsequent traversals will encounter stale pointers. - HeapWord *p = r->bottom(); - ShenandoahObjectToOopBoundedClosure objs(&cl, p, update_watermark); - // Anything beyond update_watermark is not yet allocated or initialized. - while (p < update_watermark) { - oop obj = oop(p); - objs.do_object(obj); - p += obj->size(); - } - } + update_all_references(&cl, r, update_watermark); } } } @@ -2179,6 +2165,25 @@ class ShenandoahUpdateHeapRefsTask : public AbstractGangTask { r = _regions->next(); } } + + template + void update_all_references(T* cl, ShenandoahHeapRegion* r, HeapWord* update_watermark) { + if (r->is_humongous()) { + r->oop_iterate_humongous(cl); + } else { + // We don't have liveness information about this region. + // Therefore we process all objects, rather than just marked ones. + // Otherwise subsequent traversals will encounter stale pointers. + HeapWord* p = r->bottom(); + ShenandoahObjectToOopBoundedClosure objs(cl, p, update_watermark); + // Anything beyond update_watermark is not yet allocated or initialized. + while (p < update_watermark) { + oop obj = oop(p); + objs.do_object(obj); + p += obj->size(); + } + } + } }; void ShenandoahHeap::update_heap_references(bool concurrent) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp index fa0bae48fefe5..c22cffeb8be74 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp @@ -230,10 +230,6 @@ inline HeapWord* ShenandoahHeap::allocate_from_plab(Thread* thread, size_t size) obj = allocate_from_plab_slow(thread, size); } - if (mode()->is_generational() && obj != NULL) { - ShenandoahHeap::heap()->card_scan()->register_object(obj); - } - return obj; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp index 8fceb6a2cd167..5329def70f595 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp @@ -82,10 +82,12 @@ class ShenandoahProcessOldSATB : public SATBBufferClosure { for (size_t i = 0; i < size; ++i) { oop *p = (oop *) &buffer[i]; ShenandoahHeapRegion* region = _heap->heap_region_containing(*p); - if (!region->is_trash()) { - ShenandoahMark::mark_through_ref(p, _queue, NULL, _mark_context, false); - } else { - ++_trashed_oops; + if (region->is_old()) { + if (!region->is_trash()) { + ShenandoahMark::mark_through_ref(p, _queue, NULL, _mark_context, false); + } else { + ++_trashed_oops; + } } } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp index e3604397250c3..77e68f4428233 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp @@ -594,7 +594,7 @@ inline void ShenandoahScanRemembered::oops_do(OopClosure* cl) { ShenandoahHeap* heap = ShenandoahHeap::heap(); for (size_t i = 0, n = heap->num_regions(); i < n; ++i) { ShenandoahHeapRegion* region = heap->get_region(i); - if (region->affiliation() == OLD_GENERATION) { + if (region->is_old() && region->is_active() && !region->is_cset()) { HeapWord* start_of_range = region->bottom(); HeapWord* end_of_range = region->top(); size_t start_cluster_no = cluster_for_addr(start_of_range); From fff87a60f33d60690a9335982738d968f3b9e1ab Mon Sep 17 00:00:00 2001 From: William Kemper Date: Tue, 25 May 2021 17:34:40 +0000 Subject: [PATCH 045/254] Fix verifier handling of weak references when scanning the remembered set Reviewed-by: zgu, rkennke --- .../share/gc/shenandoah/shenandoahRootVerifier.cpp | 8 ++++---- .../share/gc/shenandoah/shenandoahRootVerifier.hpp | 4 ++-- .../share/gc/shenandoah/shenandoahScanRemembered.hpp | 2 +- .../gc/shenandoah/shenandoahScanRemembered.inline.hpp | 7 ++++++- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.cpp b/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.cpp index 60b701a5d2105..8d3ac269618e8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.cpp @@ -125,7 +125,7 @@ void ShenandoahRootVerifier::oops_do(OopClosure* oops) { } } -void ShenandoahRootVerifier::roots_do(OopClosure* oops) { +void ShenandoahRootVerifier::roots_do(OopIterateClosure* oops) { ShenandoahGCStateResetter resetter; shenandoah_assert_safepoint(); @@ -140,7 +140,7 @@ void ShenandoahRootVerifier::roots_do(OopClosure* oops) { ShenandoahHeap* heap = ShenandoahHeap::heap(); if (heap->mode()->is_generational() && heap->is_gc_generation_young()) { - heap->card_scan()->oops_do(oops); + heap->card_scan()->roots_do(oops); } // Do thread roots the last. This allows verification code to find @@ -149,7 +149,7 @@ void ShenandoahRootVerifier::roots_do(OopClosure* oops) { Threads::possibly_parallel_oops_do(true, oops, &blobs); } -void ShenandoahRootVerifier::strong_roots_do(OopClosure* oops) { +void ShenandoahRootVerifier::strong_roots_do(OopIterateClosure* oops) { ShenandoahGCStateResetter resetter; shenandoah_assert_safepoint(); @@ -163,7 +163,7 @@ void ShenandoahRootVerifier::strong_roots_do(OopClosure* oops) { ShenandoahHeap* heap = ShenandoahHeap::heap(); if (heap->mode()->is_generational() && heap->is_gc_generation_young()) { - heap->card_scan()->oops_do(oops); + heap->card_scan()->roots_do(oops); } // Do thread roots the last. This allows verification code to find diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.hpp b/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.hpp index 8e5a7f98bd177..55514a747457b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.hpp @@ -64,8 +64,8 @@ class ShenandoahRootVerifier : public StackObj { void oops_do(OopClosure* cl); // Used to seed ShenandoahVerifier, do not honor root type filter - void roots_do(OopClosure* cl); - void strong_roots_do(OopClosure* cl); + void roots_do(OopIterateClosure* cl); + void strong_roots_do(OopIterateClosure* cl); static RootTypes combine(RootTypes t1, RootTypes t2); private: diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp index 85a9b16ef9d04..3cab558bacf89 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp @@ -954,7 +954,7 @@ class ShenandoahScanRemembered: public CHeapObj { // from dirty to clean and clean to dirty. The do_oops // implementations will want to update this value each time they // cross one of these boundaries. - + void roots_do(OopIterateClosure* cl); void oops_do(OopClosure* cl); }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp index 77e68f4428233..5ef784cd9596b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp @@ -591,6 +591,11 @@ class ShenandoahOopIterateAdapter : public BasicOopIterateClosure { template inline void ShenandoahScanRemembered::oops_do(OopClosure* cl) { ShenandoahOopIterateAdapter adapter(cl); + roots_do(&adapter); +} + +template +inline void ShenandoahScanRemembered::roots_do(OopIterateClosure* cl) { ShenandoahHeap* heap = ShenandoahHeap::heap(); for (size_t i = 0, n = heap->num_regions(); i < n; ++i) { ShenandoahHeapRegion* region = heap->get_region(i); @@ -604,7 +609,7 @@ inline void ShenandoahScanRemembered::oops_do(OopClosure* cl) { size_t num_clusters = (size_t) ((num_heapwords - 1 + cluster_size) / cluster_size); // Remembered set scanner - process_clusters(start_cluster_no, num_clusters, end_of_range, &adapter); + process_clusters(start_cluster_no, num_clusters, end_of_range, cl); } } } From 6f1f29a12d6b91ccb610403e9ea87fe719d0a8c8 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Wed, 26 May 2021 17:06:22 +0000 Subject: [PATCH 046/254] Encode phase for each generation separately Reviewed-by: rkennke --- .../shenandoahHeapRegionCounters.cpp | 70 +++++++++++++++---- .../shenandoahHeapRegionCounters.hpp | 14 +++- 2 files changed, 69 insertions(+), 15 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp index 4b4771cc41f8e..29da9780cf2c7 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp @@ -83,19 +83,8 @@ void ShenandoahHeapRegionCounters::update() { Atomic::cmpxchg(&_last_sample_millis, last, current) == last) { ShenandoahHeap* heap = ShenandoahHeap::heap(); - jlong status = 0; - - if (heap->is_concurrent_young_mark_in_progress()) status |= 1 << 0; - if (heap->is_concurrent_old_mark_in_progress()) status |= 1 << 1; - if (heap->is_evacuation_in_progress()) status |= 1 << 2; - if (heap->is_update_refs_in_progress()) status |= 1 << 3; - if (heap->is_degenerated_gc_in_progress()) status |= 1 << 4; - if (heap->is_full_gc_in_progress()) status |= 1 << 5; - if (heap->active_generation() != NULL) { - status |= (heap->active_generation()->generation_mode() << 6); - } - _status->set_value(status); + _status->set_value(encode_heap_status(heap)); _timestamp->set_value(os::elapsed_counter()); @@ -123,3 +112,60 @@ void ShenandoahHeapRegionCounters::update() { } } } + +static int encode_phase(ShenandoahHeap* heap) { + if (heap->is_evacuation_in_progress()) { + return 2; + } + if (heap->is_update_refs_in_progress()) { + return 3; + } + if (heap->is_concurrent_mark_in_progress()) { + return 1; + } + assert(heap->is_idle(), "What is it doing?"); + return 0; +} + +static int get_generation_shift(ShenandoahGeneration* generation) { + switch (generation->generation_mode()) { + case GLOBAL: return 0; + case OLD: return 2; + case YOUNG: return 4; + default: + ShouldNotReachHere(); + return -1; + } +} + +jlong ShenandoahHeapRegionCounters::encode_heap_status(ShenandoahHeap* heap) { + + if (heap->is_idle()) { + return 0; + } + + jlong status = 0; + if (!heap->mode()->is_generational()) { + status = encode_phase(heap); + } else { + int phase = encode_phase(heap); + ShenandoahGeneration* generation = heap->active_generation(); + assert(generation != NULL, "Expected active generation in this mode."); + int shift = get_generation_shift(generation); + status |= ((phase & 0x3) << shift); + if (heap->is_concurrent_old_mark_in_progress()) { + status |= (1 << 2); + } + log_develop_trace(gc)("%s, phase=%u, old_mark=%s, status=%zu", + generation->name(), phase, BOOL_TO_STR(heap->is_concurrent_old_mark_in_progress()), status); + } + + if (heap->is_degenerated_gc_in_progress()) { + status |= (1 << 6); + } + if (heap->is_full_gc_in_progress()) { + status |= (1 << 7); + } + + return status; +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.hpp index ebeebaa237ed6..e559cf67069f3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.hpp @@ -37,9 +37,14 @@ * * variables: * - sun.gc.shenandoah.regions.status current GC status: - * - bit 0 set when marking in progress - * - bit 1 set when evacuation in progress - * - bit 2 set when update refs in progress + * | global | old | young | mode | + * | 0..1 | 2..3 | 4..5 | 6..7 | + * + * For each generation: + * 0 = idle, 1 = marking, 2 = evacuating, 3 = updating refs + * + * For mode: + * 0 = concurrent, 1 = degenerated, 2 = full * * two variable counters per region, with $max_regions (see above) counters: * - sun.gc.shenandoah.regions.region.$i.data @@ -85,6 +90,9 @@ class ShenandoahHeapRegionCounters : public CHeapObj { ShenandoahHeapRegionCounters(); ~ShenandoahHeapRegionCounters(); void update(); + +private: + static jlong encode_heap_status(ShenandoahHeap* heap) ; }; #endif // SHARE_GC_SHENANDOAH_SHENANDOAHHEAPREGIONCOUNTERS_HPP From 9011a4b60e68d7990a5309059d3dc165c4c1e774 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Thu, 27 May 2021 15:57:53 +0000 Subject: [PATCH 047/254] Check explicitly for affiliation when updating references Reviewed-by: zgu --- src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 203385df1f828..2549461cd23f2 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -2139,8 +2139,7 @@ class ShenandoahUpdateHeapRefsTask : public AbstractGangTask { if (r->is_active() && !r->is_cset()) { if (!_heap->mode()->is_generational() || r->affiliation() == YOUNG_GENERATION) { _heap->marked_object_oop_iterate(r, &cl, update_watermark); - } else { - assert(r->affiliation() == OLD_GENERATION, "Should not be updating references on FREE regions"); + } else if (r->affiliation() == OLD_GENERATION) { if (!_heap->is_gc_generation_young() || is_mixed) { // Old region in global or mixed cycle (in which case, old regions should be marked). // We need to make sure that the next remembered set scan does not iterate over dead objects From 0b3dc74b973c48817f5e280abf8102530bdb7a4f Mon Sep 17 00:00:00 2001 From: Aleksey Shipilev Date: Mon, 14 Jun 2021 11:01:36 +0000 Subject: [PATCH 048/254] Fix 32-bit builds Reviewed-by: rkennke --- .../share/gc/shenandoah/shenandoahHeapRegionCounters.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp index 29da9780cf2c7..268516d9633e9 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp @@ -156,7 +156,7 @@ jlong ShenandoahHeapRegionCounters::encode_heap_status(ShenandoahHeap* heap) { if (heap->is_concurrent_old_mark_in_progress()) { status |= (1 << 2); } - log_develop_trace(gc)("%s, phase=%u, old_mark=%s, status=%zu", + log_develop_trace(gc)("%s, phase=%u, old_mark=%s, status=" JLONG_FORMAT, generation->name(), phase, BOOL_TO_STR(heap->is_concurrent_old_mark_in_progress()), status); } From ded735f03e4c55ebf4ee91914b99bf5ff21b29bb Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Mon, 19 Jul 2021 18:35:02 +0000 Subject: [PATCH 049/254] Do old satb during interrupting young Reviewed-by: rkennke --- .../gc/shenandoah/shenandoahConcurrentGC.cpp | 17 ++++++++++--- .../gc/shenandoah/shenandoahConcurrentGC.hpp | 3 ++- .../gc/shenandoah/shenandoahControlThread.cpp | 24 ++++++++++--------- .../gc/shenandoah/shenandoahControlThread.hpp | 2 +- .../gc/shenandoah/shenandoahGeneration.cpp | 4 ++-- .../gc/shenandoah/shenandoahMark.inline.hpp | 2 +- .../shenandoah/shenandoahYoungGeneration.cpp | 10 ++------ 7 files changed, 35 insertions(+), 27 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index adfdffb483552..d3f8fac5dc870 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -46,9 +46,10 @@ #include "prims/jvmtiTagMap.hpp" #include "utilities/events.hpp" -ShenandoahConcurrentGC::ShenandoahConcurrentGC(ShenandoahGeneration* generation) : +ShenandoahConcurrentGC::ShenandoahConcurrentGC(ShenandoahGeneration* generation, bool do_old_gc_bootstrap) : _mark(generation), _degen_point(ShenandoahDegenPoint::_degenerated_unset), + _do_old_gc_bootstrap(do_old_gc_bootstrap), _generation(generation) { } @@ -435,7 +436,11 @@ void ShenandoahConcurrentGC::op_reset() { heap->pacer()->setup_for_reset(); } - _generation->prepare_gc(); + if (_do_old_gc_bootstrap) { + heap->global_generation()->prepare_gc(); + } else { + _generation->prepare_gc(); + } } class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure { @@ -480,7 +485,13 @@ void ShenandoahConcurrentGC::op_init_mark() { _generation->set_concurrent_mark_in_progress(true); - { + if (_do_old_gc_bootstrap) { + // Update region state for both young and old regions + ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states); + ShenandoahInitMarkUpdateRegionStateClosure cl; + heap->parallel_heap_region_iterate(&cl); + } else { + // Update region state for only young regions ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states); ShenandoahInitMarkUpdateRegionStateClosure cl; _generation->parallel_heap_region_iterate(&cl); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp index c7d95b12dd836..2efcb3fb3de56 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp @@ -46,12 +46,13 @@ class ShenandoahConcurrentGC : public ShenandoahGC { private: ShenandoahConcurrentMark _mark; ShenandoahDegenPoint _degen_point; + const bool _do_old_gc_bootstrap; protected: ShenandoahGeneration* const _generation; public: - ShenandoahConcurrentGC(ShenandoahGeneration* generation); + ShenandoahConcurrentGC(ShenandoahGeneration* generation, bool do_old_gc_bootstrap = false); bool collect(GCCause::Cause cause); ShenandoahDegenPoint degen_point() const; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index 9a8b7373871b8..f5d691872f056 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -419,13 +419,13 @@ void ShenandoahControlThread::service_concurrent_normal_cycle( // they end up in, but we have to be sure we don't promote into any regions // that are in the cset (more of an issue for Milestone-8 to worry about). log_info(gc, ergo)("Start GC cycle (YOUNG)"); - service_concurrent_cycle(heap->young_generation(), cause); + service_concurrent_cycle(heap->young_generation(), cause, false); heap->young_generation()->log_status(); break; } case GLOBAL: { log_info(gc, ergo)("Start GC cycle (GLOBAL)"); - service_concurrent_cycle(heap->global_generation(), cause); + service_concurrent_cycle(heap->global_generation(), cause, false); heap->global_generation()->log_status(); break; } @@ -453,14 +453,10 @@ void ShenandoahControlThread::service_concurrent_old_cycle(const ShenandoahHeap* assert(old_generation->task_queues()->is_empty(), "Old mark queues should be empty."); young_generation->set_old_gen_task_queues(old_generation->task_queues()); - + young_generation->set_mark_incomplete(); old_generation->set_mark_incomplete(); - service_concurrent_cycle(young_generation, cause); - - // Young generation no longer needs this reference to the old concurrent - // mark so clean it up. - young_generation->set_old_gen_task_queues(NULL); + service_concurrent_cycle(young_generation, cause, true); if (!heap->cancelled_gc()) { // Reset the degenerated point. Normally this would happen at the top @@ -518,7 +514,9 @@ void ShenandoahControlThread::resume_concurrent_old_cycle(ShenandoahGeneration* // precisely where the regulator is allowed to cancel a GC. ShenandoahOldGC gc(generation, _allow_old_preemption); if (gc.collect(cause)) { - // Cycle is complete + // Old collection is complete, the young generation no longer needs this + // reference to the old concurrent mark so clean it up. + heap->young_generation()->set_old_gen_task_queues(NULL); generation->heuristics()->record_success_concurrent(); heap->shenandoah_policy()->record_success_concurrent(); } @@ -537,7 +535,7 @@ void ShenandoahControlThread::resume_concurrent_old_cycle(ShenandoahGeneration* } } -void ShenandoahControlThread::service_concurrent_cycle(ShenandoahGeneration* generation, GCCause::Cause cause) { +void ShenandoahControlThread::service_concurrent_cycle(ShenandoahGeneration* generation, GCCause::Cause cause, bool do_old_gc_bootstrap) { // Normal cycle goes via all concurrent phases. If allocation failure (af) happens during // any of the concurrent phases, it first degrades to Degenerated GC and completes GC there. // If second allocation failure happens during Degenerated GC cycle (for example, when GC @@ -581,7 +579,7 @@ void ShenandoahControlThread::service_concurrent_cycle(ShenandoahGeneration* gen TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); - ShenandoahConcurrentGC gc(generation); + ShenandoahConcurrentGC gc(generation, do_old_gc_bootstrap); if (gc.collect(cause)) { // Cycle is complete generation->heuristics()->record_success_concurrent(); @@ -654,6 +652,10 @@ void ShenandoahControlThread::service_stw_degenerated_cycle(GCCause::Cause cause ShenandoahGCSession session(cause, _degen_generation); ShenandoahDegenGC gc(point, _degen_generation); + + // Just in case degenerated cycle preempted old-gen marking, clear the old-gen task queues. + heap->young_generation()->set_old_gen_task_queues(NULL); + gc.collect(cause); assert(heap->young_generation()->task_queues()->is_empty(), "Unexpected young generation marking tasks"); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp index 6fdc479252147..40341d3d5b6f0 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp @@ -103,7 +103,7 @@ class ShenandoahControlThread: public ConcurrentGCThread { bool check_cancellation_or_degen(ShenandoahGC::ShenandoahDegenPoint point); void resume_concurrent_old_cycle(ShenandoahGeneration* generation, GCCause::Cause cause); - void service_concurrent_cycle(ShenandoahGeneration* generation, GCCause::Cause cause); + void service_concurrent_cycle(ShenandoahGeneration* generation, GCCause::Cause cause, bool reset_old_bitmap_specially); void service_stw_full_cycle(GCCause::Cause cause); void service_stw_degenerated_cycle(GCCause::Cause cause, ShenandoahGC::ShenandoahDegenPoint point); void service_uncommit(double shrink_before, size_t shrink_until); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index 947526252dfd5..eb1ab6c3e2c96 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -136,8 +136,10 @@ void ShenandoahGeneration::reset_mark_bitmap() { } void ShenandoahGeneration::prepare_gc() { + // Reset mark bitmap for this generation (typically young) reset_mark_bitmap(); + // Capture Top At Mark Start for this generation (typically young) ShenandoahResetUpdateRegionStateClosure cl; parallel_heap_region_iterate(&cl); } @@ -248,8 +250,6 @@ void ShenandoahGeneration::scan_remembered_set() { assert(generation_mode() == YOUNG, "Should only scan remembered set for young generation."); ShenandoahHeap* const heap = ShenandoahHeap::heap(); - // TODO: Add a phase for rset scan. - // ShenandoahGCPhase phase(ShenandoahPhaseTimings::finish_mark); uint nworkers = heap->workers()->active_workers(); reserve_task_queues(nworkers); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp index 125df2774e9b3..a3532a1bf048c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp @@ -269,7 +269,7 @@ inline void ShenandoahMark::mark_through_ref(T *p, ShenandoahObjToScanQueue* q, mark_ref(q, mark_context, weak, obj); shenandoah_assert_marked(p, obj); } else if (old != nullptr) { - // Young mark, bootstrapping old. + // Young mark, bootstrapping old or concurrent with old marking. mark_ref(old, mark_context, weak, obj); shenandoah_assert_marked(p, obj); } else if (GENERATION == OLD) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp index b724ee808d121..d21cdd49d46b4 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp @@ -113,14 +113,8 @@ bool ShenandoahYoungGeneration::contains(ShenandoahHeapRegion* region) const { } void ShenandoahYoungGeneration::parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl) { - if (_old_gen_task_queues != NULL) { - // No generation filter on regions, we need to iterate all the regions. - ShenandoahHeap::heap()->parallel_heap_region_iterate(cl); - } else { - // Just the young generations here. - ShenandoahGenerationRegionClosure young_regions(cl); - ShenandoahHeap::heap()->parallel_heap_region_iterate(&young_regions); - } + ShenandoahGenerationRegionClosure young_regions(cl); + ShenandoahHeap::heap()->parallel_heap_region_iterate(&young_regions); } void ShenandoahYoungGeneration::heap_region_iterate(ShenandoahHeapRegionClosure* cl) { From d775516e9b2216735fa059f8e986c7a8d280c9f0 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Thu, 22 Jul 2021 14:28:54 +0000 Subject: [PATCH 050/254] Concurrent remembered set scanning Reviewed-by: rkennke --- .../heuristics/shenandoahHeuristics.cpp | 9 +- .../heuristics/shenandoahHeuristics.hpp | 3 +- .../heuristics/shenandoahOldHeuristics.cpp | 14 +- .../heuristics/shenandoahOldHeuristics.hpp | 6 +- .../gc/shenandoah/shenandoahBarrierSet.cpp | 2 +- .../gc/shenandoah/shenandoahCardTable.cpp | 69 +++- .../gc/shenandoah/shenandoahCardTable.hpp | 47 ++- .../gc/shenandoah/shenandoahConcurrentGC.cpp | 38 +- .../gc/shenandoah/shenandoahConcurrentGC.hpp | 9 +- .../gc/shenandoah/shenandoahControlThread.cpp | 4 +- .../gc/shenandoah/shenandoahDegeneratedGC.cpp | 2 +- .../share/gc/shenandoah/shenandoahFreeSet.cpp | 37 +- .../share/gc/shenandoah/shenandoahFreeSet.hpp | 7 + .../share/gc/shenandoah/shenandoahFullGC.cpp | 60 +-- .../gc/shenandoah/shenandoahGeneration.cpp | 66 +++- .../gc/shenandoah/shenandoahGeneration.hpp | 10 +- .../shenandoah/shenandoahGlobalGeneration.cpp | 4 +- .../shenandoah/shenandoahGlobalGeneration.hpp | 2 +- .../share/gc/shenandoah/shenandoahHeap.cpp | 319 ++++++++++++++-- .../share/gc/shenandoah/shenandoahHeap.hpp | 14 + .../gc/shenandoah/shenandoahHeap.inline.hpp | 22 +- .../gc/shenandoah/shenandoahHeapRegion.cpp | 262 ++++++++----- .../gc/shenandoah/shenandoahHeapRegion.hpp | 29 +- .../shenandoahHeapRegion.inline.hpp | 4 +- .../share/gc/shenandoah/shenandoahMark.cpp | 16 +- .../share/gc/shenandoah/shenandoahMark.hpp | 2 + .../gc/shenandoah/shenandoahMark.inline.hpp | 37 +- .../gc/shenandoah/shenandoahMarkBitMap.cpp | 25 +- .../gc/shenandoah/shenandoahMarkBitMap.hpp | 2 + .../gc/shenandoah/shenandoahMarkClosures.cpp | 7 + .../gc/shenandoah/shenandoahMarkClosures.hpp | 12 + .../shenandoah/shenandoahMarkingContext.cpp | 47 ++- .../shenandoah/shenandoahMarkingContext.hpp | 4 +- .../shenandoahMarkingContext.inline.hpp | 45 ++- .../share/gc/shenandoah/shenandoahOldGC.cpp | 45 ++- .../share/gc/shenandoah/shenandoahOldGC.hpp | 5 + .../gc/shenandoah/shenandoahOopClosures.hpp | 19 + .../shenandoahOopClosures.inline.hpp | 18 + .../shenandoahReferenceProcessor.cpp | 12 +- .../shenandoah/shenandoahScanRemembered.cpp | 3 +- .../shenandoah/shenandoahScanRemembered.hpp | 133 +++++-- .../shenandoahScanRemembered.inline.hpp | 356 ++++++++++++++---- .../gc/shenandoah/shenandoahVMOperations.hpp | 6 +- .../gc/shenandoah/shenandoahVerifier.cpp | 34 +- .../gc/shenandoah/shenandoahVerifier.hpp | 20 +- .../shenandoah/shenandoahYoungGeneration.cpp | 19 +- 46 files changed, 1535 insertions(+), 371 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp index ae21f753df2ba..1b4c28354c028 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp @@ -72,7 +72,9 @@ ShenandoahHeuristics::~ShenandoahHeuristics() { FREE_C_HEAP_ARRAY(RegionGarbage, _region_data); } -void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collection_set, ShenandoahOldHeuristics* old_heuristics) { +// Returns true iff the chosen collection set includes old-gen regions +bool ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collection_set, ShenandoahOldHeuristics* old_heuristics) { + bool result = false; ShenandoahHeap* heap = ShenandoahHeap::heap(); assert(collection_set->count() == 0, "Must be empty"); @@ -162,7 +164,9 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec if (immediate_percent <= ShenandoahImmediateThreshold) { if (old_heuristics != NULL) { - old_heuristics->prime_collection_set(collection_set); + if (old_heuristics->prime_collection_set(collection_set)) { + result = true; + } } // else, this is global collection and doesn't need to prime_collection_set @@ -191,6 +195,7 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec byte_size_in_proper_unit(collection_set->garbage()), proper_unit_for_byte_size(collection_set->garbage()), cset_percent); + return result; } void ShenandoahHeuristics::record_cycle_start() { diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp index 3cebd71feda4f..65885b9a9779a 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp @@ -140,7 +140,8 @@ class ShenandoahHeuristics : public CHeapObj { virtual void record_requested_gc(); - virtual void choose_collection_set(ShenandoahCollectionSet* collection_set, ShenandoahOldHeuristics* old_heuristics); + // Return true iff the chosen collection set includes at least one old-gen region. + virtual bool choose_collection_set(ShenandoahCollectionSet* collection_set, ShenandoahOldHeuristics* old_heuristics); virtual bool can_unload_classes(); virtual bool can_unload_classes_normal(); diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp index 1fd737d1061f3..c9ab820507acb 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp @@ -38,7 +38,7 @@ ShenandoahOldHeuristics::ShenandoahOldHeuristics(ShenandoahGeneration* generatio { } -void ShenandoahOldHeuristics::prime_collection_set(ShenandoahCollectionSet* collection_set) { +bool ShenandoahOldHeuristics::prime_collection_set(ShenandoahCollectionSet* collection_set) { uint included_old_regions = 0; size_t evacuated_old_bytes = 0; @@ -141,15 +141,15 @@ void ShenandoahOldHeuristics::prime_collection_set(ShenandoahCollectionSet* coll (unsigned long long) included_old_regions, (unsigned long long) evacuated_old_bytes); } + return (included_old_regions > 0); } - -void ShenandoahOldHeuristics::choose_collection_set(ShenandoahCollectionSet* collection_set, ShenandoahOldHeuristics* old_heuristics) { - assert(collection_set->count() == 0, "Must be empty"); - +// Both arguments are don't cares for old-gen collections +bool ShenandoahOldHeuristics::choose_collection_set(ShenandoahCollectionSet* collection_set, ShenandoahOldHeuristics* old_heuristics) { // Old-gen doesn't actually choose a collection set to be evacuated by its own gang of worker tasks. // Instead, it computes the set of regions to be evacuated by subsequent young-gen evacuation passes. prepare_for_old_collections(); + return false; } void ShenandoahOldHeuristics::prepare_for_old_collections() { @@ -212,7 +212,7 @@ void ShenandoahOldHeuristics::prepare_for_old_collections() { // Note that we do not coalesce and fill occupied humongous regions // HR: humongous regions, RR: regular regions, CF: coalesce and fill regions - log_info(gc)("Old-gen mark evac (%llu RR), %llu CF)", + log_info(gc)("Old-gen mark evac (%llu RR, %llu CF)", (unsigned long long) (_hidden_old_collection_candidates), (unsigned long long) _old_coalesce_and_fill_candidates); return; @@ -227,7 +227,7 @@ void ShenandoahOldHeuristics::prepare_for_old_collections() { // Note that we do not coalesce and fill occupied humongous regions // HR: humongous regions, RR: regular regions, CF: coalesce and fill regions - log_info(gc)("Old-gen mark evac (%llu RR), %llu CF)", + log_info(gc)("Old-gen mark evac (%llu RR, %llu CF)", (unsigned long long) (_hidden_old_collection_candidates), (unsigned long long) _old_coalesce_and_fill_candidates); } diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp index 26f975305d480..f55fe46b7fef2 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp @@ -69,9 +69,11 @@ class ShenandoahOldHeuristics : public ShenandoahHeuristics { public: ShenandoahOldHeuristics(ShenandoahGeneration* generation); - virtual void choose_collection_set(ShenandoahCollectionSet* collection_set, ShenandoahOldHeuristics* old_heuristics); + // Return true iff chosen collection set includes at least one old-gen HeapRegion. + virtual bool choose_collection_set(ShenandoahCollectionSet* collection_set, ShenandoahOldHeuristics* old_heuristics); - void prime_collection_set(ShenandoahCollectionSet* set); + // Return true iff the collection set is primed with at least one old-gen region. + bool prime_collection_set(ShenandoahCollectionSet* set); // Having coalesced and filled all old-gen heap regions that are not part of the old-gen collection set, begin // evacuating the collection set. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp index d26eeb48695af..8c68b56d7c3f8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp @@ -158,6 +158,6 @@ void ShenandoahBarrierSet::write_ref_array(HeapWord* start, size_t count) { // If compressed oops were not being used, these should already be aligned assert(UseCompressedOops || (aligned_start == start && aligned_end == end), "Expected heap word alignment of start and end"); - card_table()->dirty_MemRegion(MemRegion(aligned_start, aligned_end)); + _heap->card_scan()->mark_range_as_dirty(aligned_start, (aligned_end - aligned_start)); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCardTable.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCardTable.cpp index 0388175a89a40..f00e924f3776a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCardTable.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCardTable.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, Amazon.com, Inc. and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2021, Amazon.com, Inc. and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -23,11 +23,44 @@ */ #include "precompiled.hpp" -#include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahCardTable.hpp" +#include "gc/shenandoah/shenandoahHeap.inline.hpp" +#include "gc/shenandoah/shenandoahUtils.hpp" void ShenandoahCardTable::initialize() { CardTable::initialize(); + _write_byte_map = _byte_map; + _write_byte_map_base = _byte_map_base; + const size_t rs_align = _page_size == (size_t) os::vm_page_size() ? 0 : + MAX2(_page_size, (size_t) os::vm_allocation_granularity()); + + ReservedSpace heap_rs(_byte_map_size, rs_align, false); + if (!heap_rs.is_reserved()) { + vm_exit_during_initialization("Could not reserve enough space for second copy of card marking array"); + } + os::commit_memory_or_exit(heap_rs.base(), _byte_map_size, rs_align, false, "Cannot commit memory for second copy of card table"); + + HeapWord* low_bound = _whole_heap.start(); + _read_byte_map = (CardValue*) heap_rs.base(); + _read_byte_map_base = _read_byte_map - (uintptr_t(low_bound) >> card_shift); + + log_trace(gc, barrier)("ShenandoahCardTable::ShenandoahCardTable: "); + log_trace(gc, barrier)(" &_read_byte_map[0]: " INTPTR_FORMAT " &_read_byte_map[_last_valid_index]: " INTPTR_FORMAT, + p2i(&_read_byte_map[0]), p2i(&_read_byte_map[_last_valid_index])); + log_trace(gc, barrier)(" _read_byte_map_base: " INTPTR_FORMAT, p2i(_read_byte_map_base)); + + // TODO: As currently implemented, we do not swap pointers between _read_byte_map and _write_byte_map + // because the mutator write barrier hard codes the address of the _write_byte_map_base. Instead, + // the current implementation simply copies contents of _write_byte_map onto _read_byte_map and cleans + // the entirety of _write_byte_map at the init_mark safepoint. + // + // If we choose to modify the mutator write barrier so that we can swap _read_byte_map_base and + // _write_byte_map_base pointers, we may also have to figure out certain details about how the + // _guard_region is implemented so that we can replicate the read and write versions of this region. + // + // Alternatively, we may switch to a SATB-based write barrier and replace the direct card-marking + // remembered set with something entirely different. + resize_covered_region(_whole_heap); } @@ -48,3 +81,35 @@ bool ShenandoahCardTable::is_dirty(MemRegion mr) { void ShenandoahCardTable::clear() { CardTable::clear(_whole_heap); } + +// TODO: This service is not currently used because we are not able to swap _read_byte_map_base and +// _write_byte_map_base pointers. If we were able to do so, we would invoke clear_read_table "immediately" +// following the end of concurrent remembered set scanning so that this read card table would be ready +// to serve as the new write card table at the time these pointer values were next swapped. +// +// In the current implementation, the write-table is cleared immediately after its contents is copied to +// the read table, obviating the need for this service. +void ShenandoahCardTable::clear_read_table() { + for (size_t i = 0; i < _byte_map_size; i++) { + _read_byte_map[i] = clean_card; + } +} + +// TODO: This service is not currently used because the mutator write barrier implementation hard codes the +// location of the _write_byte_may_base. If we change the mutator's write barrier implementation, then we +// may use this service to exchange the roles of the read-card-table and write-card-table. +void ShenandoahCardTable::swap_card_tables() { + shenandoah_assert_safepoint(); + + CardValue* save_value = _read_byte_map; + _read_byte_map = _write_byte_map; + _write_byte_map = save_value; + + save_value = _read_byte_map_base; + _read_byte_map_base = _write_byte_map_base; + _write_byte_map_base = save_value; + + // update the superclass instance variables + _byte_map = _write_byte_map; + _byte_map_base = _write_byte_map_base; +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCardTable.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCardTable.hpp index 98e944e662c1d..b8013561e24a0 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCardTable.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCardTable.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, Amazon.com, Inc. and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2021, Amazon.com, Inc. and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -33,8 +33,30 @@ class ShenandoahCardTable: public CardTable { friend class VMStructs; +protected: + // We maintain two copies of the card table to facilitate concurrent remembered set scanning + // and concurrent clearing of stale remembered set information. During the init_mark safepoint, + // we copy the contents of _write_byte_map to _read_byte_map and clear _write_byte_map. + // + // Concurrent remembered set scanning reads from _read_byte_map while concurrent mutator write + // barriers are overwriting cards of the _write_byte_map with DIRTY codes. Concurrent remembered + // set scanning also overwrites cards of the _write_byte_map with DIRTY codes whenever it discovers + // interesting pointers. + // + // During a concurrent update-references phase, we scan the _write_byte_map concurrently to find + // all old-gen references that may need to be updated. + // + // In a future implementation, we may swap the values of _read_byte_map and _write_byte_map during + // the init-mark safepoint to avoid the need for bulk STW copying and initialization. Doing so + // requires a change to the implementation of mutator write barriers as the address of the card + // table is currently in-lined and hard-coded. + CardValue* _read_byte_map; + CardValue* _write_byte_map; + CardValue* _read_byte_map_base; + CardValue* _write_byte_map_base; + public: - ShenandoahCardTable(MemRegion whole_heap): CardTable(whole_heap) { } + ShenandoahCardTable(MemRegion whole_heap) : CardTable(whole_heap) { } virtual void initialize(); @@ -43,6 +65,27 @@ class ShenandoahCardTable: public CardTable { bool is_dirty(MemRegion mr); void clear(); + + void clear_read_table(); + + // Exchange the roles of the read and write card tables. + void swap_card_tables(); + + CardValue* read_byte_map() { + return _read_byte_map; + } + + CardValue* write_byte_map() { + return _write_byte_map; + } + + CardValue* read_byte_map_base() { + return _read_byte_map_base; + } + + CardValue* write_byte_map_base() { + return _write_byte_map_base; + } }; #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHCARDTABLE_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index d3f8fac5dc870..48aac46e2fb5e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -49,6 +49,7 @@ ShenandoahConcurrentGC::ShenandoahConcurrentGC(ShenandoahGeneration* generation, bool do_old_gc_bootstrap) : _mark(generation), _degen_point(ShenandoahDegenPoint::_degenerated_unset), + _mixed_evac (false), _do_old_gc_bootstrap(do_old_gc_bootstrap), _generation(generation) { } @@ -66,7 +67,12 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) { // Start initial mark under STW vmop_entry_init_mark(); - // Concurrent mark roots + // Concurrent remembered set scanning + if (_generation->generation_mode() == YOUNG) { + _generation->scan_remembered_set(); + } + + // Concurrent mark roots entry_mark_roots(); if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_outside_cycle)) return false; @@ -148,7 +154,7 @@ void ShenandoahConcurrentGC::vmop_entry_init_mark() { ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_mark_gross); heap->try_inject_alloc_failure(); - VM_ShenandoahInitMark op(this); + VM_ShenandoahInitMark op(this, _do_old_gc_bootstrap); VMThread::execute(&op); // jump to entry_init_mark() under safepoint } @@ -192,6 +198,12 @@ void ShenandoahConcurrentGC::entry_init_mark() { ShenandoahWorkerPolicy::calc_workers_for_init_marking(), "init marking"); + if (ShenandoahHeap::heap()->mode()->is_generational() && (_generation->generation_mode() == YOUNG)) { + // The current implementation of swap_remembered_set() copies the write-card-table + // to the read-card-table. + _generation->swap_remembered_set(); + } + op_init_mark(); } @@ -435,12 +447,7 @@ void ShenandoahConcurrentGC::op_reset() { if (ShenandoahPacing) { heap->pacer()->setup_for_reset(); } - - if (_do_old_gc_bootstrap) { - heap->global_generation()->prepare_gc(); - } else { - _generation->prepare_gc(); - } + _generation->prepare_gc(_do_old_gc_bootstrap); } class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure { @@ -453,7 +460,8 @@ class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionCl assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index()); if (r->is_active()) { // Check if region needs updating its TAMS. We have updated it already during concurrent - // reset, so it is very likely we don't need to do another write here. + // reset, so it is very likely we don't need to do another write here. Since most regions + // are not "active", this path is relatively rare. if (_ctx->top_at_mark_start(r) != r->top()) { _ctx->capture_top_at_mark_start(r); } @@ -490,6 +498,7 @@ void ShenandoahConcurrentGC::op_init_mark() { ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states); ShenandoahInitMarkUpdateRegionStateClosure cl; heap->parallel_heap_region_iterate(&cl); + heap->old_generation()->parallel_heap_region_iterate(&cl); } else { // Update region state for only young regions ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states); @@ -505,10 +514,6 @@ void ShenandoahConcurrentGC::op_init_mark() { // Make above changes visible to worker threads OrderAccess::fence(); - if (_generation->generation_mode() == YOUNG) { - _generation->scan_remembered_set(); - } - // Arm nmethods for concurrent marking. When a nmethod is about to be executed, // we need to make sure that all its metadata are marked. alternative is to remark // thread roots at final mark pause, but it can be potential latency killer. @@ -546,7 +551,8 @@ void ShenandoahConcurrentGC::op_final_mark() { // Notify JVMTI that the tagmap table will need cleaning. JvmtiTagMap::set_needs_cleaning(); - _generation->prepare_regions_and_collection_set(true /*concurrent*/); + bool mixed_evac = _generation->prepare_regions_and_collection_set(true /*concurrent*/); + heap->set_mixed_evac(mixed_evac); // Has to be done after cset selection heap->prepare_concurrent_roots(); @@ -909,7 +915,9 @@ void ShenandoahConcurrentGC::op_init_updaterefs() { heap->set_evacuation_in_progress(false); heap->prepare_update_heap_references(true /*concurrent*/); heap->set_update_refs_in_progress(true); - + if (ShenandoahVerify) { + heap->verifier()->verify_before_updaterefs(); + } if (ShenandoahPacing) { heap->pacer()->setup_for_updaterefs(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp index 2efcb3fb3de56..8dbb6ae40b991 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp @@ -43,16 +43,19 @@ class ShenandoahConcurrentGC : public ShenandoahGC { friend class VM_ShenandoahInitUpdateRefs; friend class VM_ShenandoahFinalUpdateRefs; -private: +protected: ShenandoahConcurrentMark _mark; + +private: ShenandoahDegenPoint _degen_point; + bool _mixed_evac; // true iff most recent evacuation includes old-gen HeapRegions const bool _do_old_gc_bootstrap; protected: ShenandoahGeneration* const _generation; public: - ShenandoahConcurrentGC(ShenandoahGeneration* generation, bool do_old_gc_bootstrap = false); + ShenandoahConcurrentGC(ShenandoahGeneration* generation, bool do_old_gc_bootstrap); bool collect(GCCause::Cause cause); ShenandoahDegenPoint degen_point() const; @@ -89,6 +92,7 @@ class ShenandoahConcurrentGC : public ShenandoahGC { void entry_strong_roots(); void entry_cleanup_early(); void entry_rendezvous_roots(); + virtual void op_final_mark(); private: void entry_evacuate(); @@ -101,7 +105,6 @@ class ShenandoahConcurrentGC : public ShenandoahGC { void op_init_mark(); void op_mark_roots(); void op_mark(); - void op_final_mark(); void op_thread_roots(); void op_weak_refs(); void op_weak_roots(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index f5d691872f056..f0b6731cd7440 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -417,7 +417,7 @@ void ShenandoahControlThread::service_concurrent_normal_cycle( // concurrent mark in the old generation. We need to think about promotions // in this case. Promoted objects should be above the TAMS in the old regions // they end up in, but we have to be sure we don't promote into any regions - // that are in the cset (more of an issue for Milestone-8 to worry about). + // that are in the cset. log_info(gc, ergo)("Start GC cycle (YOUNG)"); service_concurrent_cycle(heap->young_generation(), cause, false); heap->young_generation()->log_status(); @@ -455,9 +455,7 @@ void ShenandoahControlThread::service_concurrent_old_cycle(const ShenandoahHeap* young_generation->set_old_gen_task_queues(old_generation->task_queues()); young_generation->set_mark_incomplete(); old_generation->set_mark_incomplete(); - service_concurrent_cycle(young_generation, cause, true); - if (!heap->cancelled_gc()) { // Reset the degenerated point. Normally this would happen at the top // of the control loop, but here we have just completed a young cycle diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp index 94a8aa083f1d8..2d7adc641158a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp @@ -222,7 +222,7 @@ void ShenandoahDegenGC::op_degenerated() { } void ShenandoahDegenGC::op_reset() { - _generation->prepare_gc(); + _generation->prepare_gc(false); } void ShenandoahDegenGC::op_mark() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index 2868f9df43396..63b29d13ef64c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -167,9 +167,23 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah try_recycle_trashed(r); if (r->affiliation() == ShenandoahRegionAffiliation::FREE) { - // This free region might have garbage in its remembered set representation. - _heap->clear_cards_for(r); + ShenandoahMarkingContext* const ctx = _heap->complete_marking_context(); + if (req.affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) { + // This free region might have garbage in its remembered set representation. + _heap->clear_cards_for(r); + } r->set_affiliation(req.affiliation()); + r->set_update_watermark(r->bottom()); + ctx->capture_top_at_mark_start(r); + + assert(ctx->top_at_mark_start(r) == r->bottom(), "Newly established allocation region starts with TAMS equal to bottom"); + assert(ctx->is_bitmap_clear_range(ctx->top_bitmap(r), r->end()), "Bitmap above top_bitmap() must be clear"); + + // Leave top_bitmap alone. The first time a heap region is put into service, top_bitmap should equal end. + // Thereafter, it should represent the upper bound on parts of the bitmap that need to be cleared. + log_debug(gc)("NOT clearing bitmap for region " SIZE_FORMAT ", top_bitmap: " + PTR_FORMAT " at transition from FREE to %s", + r->index(), p2i(ctx->top_bitmap(r)), affiliation_name(req.affiliation())); } else if (r->affiliation() != req.affiliation()) { return NULL; } @@ -313,6 +327,7 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req) { }; size_t remainder = words_size & ShenandoahHeapRegion::region_size_words_mask(); + ShenandoahMarkingContext* const ctx = _heap->complete_marking_context(); // Initialize regions: for (size_t i = beg; i <= end; i++) { @@ -336,8 +351,20 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req) { used_words = ShenandoahHeapRegion::region_size_words(); } - r->set_top(r->bottom() + used_words); r->set_affiliation(req.affiliation()); + r->set_update_watermark(r->bottom()); + r->set_top(r->bottom()); // Set top to bottom so we can capture TAMS + ctx->capture_top_at_mark_start(r); + r->set_top(r->bottom() + used_words); // Then change top to reflect allocation of humongous object. + assert(ctx->top_at_mark_start(r) == r->bottom(), "Newly established allocation region starts with TAMS equal to bottom"); + assert(ctx->is_bitmap_clear_range(ctx->top_bitmap(r), r->end()), "Bitmap above top_bitmap() must be clear"); + + // Leave top_bitmap alone. The first time a heap region is put into service, top_bitmap should equal end. + // Thereafter, it should represent the upper bound on parts of the bitmap that need to be cleared. + // ctx->clear_bitmap(r); + log_debug(gc)("NOT clearing bitmap for Humongous region [" PTR_FORMAT ", " PTR_FORMAT "], top_bitmap: " + PTR_FORMAT " at transition from FREE to %s", + p2i(r->bottom()), p2i(r->end()), p2i(ctx->top_bitmap(r)), affiliation_name(req.affiliation())); _mutator_free_bitmap.clear_bit(r->index()); } @@ -448,6 +475,7 @@ void ShenandoahFreeSet::rebuild() { shenandoah_assert_heaplocked(); clear(); + log_debug(gc)("Rebuilding FreeSet"); for (size_t idx = 0; idx < _heap->num_regions(); idx++) { ShenandoahHeapRegion* region = _heap->get_region(idx); if (region->is_alloc_allowed() || region->is_trash()) { @@ -461,6 +489,8 @@ void ShenandoahFreeSet::rebuild() { assert(!is_mutator_free(idx), "We are about to add it, it shouldn't be there already"); _mutator_free_bitmap.set_bit(idx); + + log_debug(gc)(" Setting _mutator_free_bitmap bit for " SIZE_FORMAT, idx); } } @@ -478,6 +508,7 @@ void ShenandoahFreeSet::rebuild() { size_t ac = alloc_capacity(region); _capacity -= ac; reserved += ac; + log_debug(gc)(" Shifting region " SIZE_FORMAT " from mutator_free to collector_free", idx); } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp index a1a8a9634546f..e59df83b28c4b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp @@ -50,6 +50,13 @@ class ShenandoahFreeSet : public CHeapObj { HeapWord* try_allocate_in(ShenandoahHeapRegion* region, ShenandoahAllocRequest& req, bool& in_new_region); HeapWord* allocate_with_affiliation(ShenandoahRegionAffiliation affiliation, ShenandoahAllocRequest& req, bool& in_new_region); + + // While holding the heap lock, allocate memory for a single object which is to be entirely contained + // within a single HeapRegion as characterized by req. The req.size() value is known to be less than or + // equal to ShenandoahHeapRegion::humongous_threshold_words(). The caller of allocate_single is responsible + // for registering the resulting object and setting the remembered set card values as appropriate. The + // most common case is that we are allocating a PLAB in which case object registering and card dirtying + // is managed after the PLAB is divided into individual objects. HeapWord* allocate_single(ShenandoahAllocRequest& req, bool& in_new_region); HeapWord* allocate_contiguous(ShenandoahAllocRequest& req); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index 19f48700f1ac9..00c71c52088cc 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -120,6 +120,9 @@ void ShenandoahFullGC::op_full(GCCause::Cause cause) { void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) { ShenandoahHeap* heap = ShenandoahHeap::heap(); + if (ShenandoahHeap::heap()->mode()->is_generational()) { + fatal("Full GC not yet supported for generational mode in do_it()."); + } if (ShenandoahVerify) { heap->verifier()->verify_before_fullgc(); } @@ -196,6 +199,7 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) { } if (UseTLAB) { + // TODO: Do we need to explicitly retire PLABs? heap->gclabs_retire(ResizeTLAB); heap->tlabs_retire(ResizeTLAB); } @@ -275,8 +279,10 @@ class ShenandoahPrepareForMarkClosure: public ShenandoahHeapRegionClosure { ShenandoahPrepareForMarkClosure() : _ctx(ShenandoahHeap::heap()->marking_context()) {} void heap_region_do(ShenandoahHeapRegion *r) { - _ctx->capture_top_at_mark_start(r); - r->clear_live_data(); + if (r->affiliation() != FREE) { + _ctx->capture_top_at_mark_start(r); + r->clear_live_data(); + } } bool is_thread_safe() { return true; } @@ -467,6 +473,7 @@ void ShenandoahFullGC::calculate_target_humongous_objects() { size_t to_begin = heap->num_regions(); size_t to_end = heap->num_regions(); + log_debug(gc)("Full GC calculating target humongous objects from end " SIZE_FORMAT, to_end); for (size_t c = heap->num_regions(); c > 0; c--) { ShenandoahHeapRegion *r = heap->get_region(c - 1); if (r->is_humongous_continuation() || (r->new_top() == r->bottom())) { @@ -533,25 +540,32 @@ class ShenandoahTrashImmediateGarbageClosure: public ShenandoahHeapRegionClosure _ctx(ShenandoahHeap::heap()->complete_marking_context()) {} void heap_region_do(ShenandoahHeapRegion* r) { - if (r->is_humongous_start()) { - oop humongous_obj = oop(r->bottom()); - if (!_ctx->is_marked(humongous_obj)) { - assert(!r->has_live(), - "Region " SIZE_FORMAT " is not marked, should not have live", r->index()); - _heap->trash_humongous_region_at(r); - } else { - assert(r->has_live(), - "Region " SIZE_FORMAT " should have live", r->index()); - } - } else if (r->is_humongous_continuation()) { - // If we hit continuation, the non-live humongous starts should have been trashed already - assert(r->humongous_start_region()->has_live(), - "Region " SIZE_FORMAT " should have live", r->index()); - } else if (r->is_regular()) { - if (!r->has_live()) { - r->make_trash_immediate(); + if (r->affiliation() != FREE) { + if (r->is_humongous_start()) { + oop humongous_obj = oop(r->bottom()); + if (!_ctx->is_marked(humongous_obj)) { + assert(!r->has_live(), + "Humongous Start %s Region " SIZE_FORMAT " is not marked, should not have live", + affiliation_name(r->affiliation()), r->index()); + log_debug(gc)("Trashing immediate humongous region " SIZE_FORMAT " because not marked", r->index()); + _heap->trash_humongous_region_at(r); + } else { + assert(r->has_live(), + "Humongous Start %s Region " SIZE_FORMAT " should have live", affiliation_name(r->affiliation()), r->index()); + } + } else if (r->is_humongous_continuation()) { + // If we hit continuation, the non-live humongous starts should have been trashed already + assert(r->humongous_start_region()->has_live(), + "Humongous Continuation %s Region " SIZE_FORMAT " should have live", affiliation_name(r->affiliation()), r->index()); + } else if (r->is_regular()) { + if (!r->has_live()) { + log_debug(gc)("Trashing immediate regular region " SIZE_FORMAT " because has no live", r->index()); + r->make_trash_immediate(); + } } } + // else, ignore this FREE region. + // TODO: change iterators so they do not process FREE regions. } }; @@ -971,6 +985,9 @@ void ShenandoahFullGC::compact_humongous_objects() { assert(old_start != new_start, "must be real move"); assert(r->is_stw_move_allowed(), "Region " SIZE_FORMAT " should be movable", r->index()); + log_debug(gc)("Full GC compaction moves humongous object from region " SIZE_FORMAT " to region " SIZE_FORMAT, + old_start, new_start); + Copy::aligned_conjoint_words(heap->get_region(old_start)->bottom(), heap->get_region(new_start)->bottom(), words_size); @@ -979,6 +996,7 @@ void ShenandoahFullGC::compact_humongous_objects() { new_obj->init_mark(); { + ShenandoahRegionAffiliation original_affiliation = r->affiliation(); for (size_t c = old_start; c <= old_end; c++) { ShenandoahHeapRegion* r = heap->get_region(c); r->make_regular_bypass(); @@ -988,9 +1006,9 @@ void ShenandoahFullGC::compact_humongous_objects() { for (size_t c = new_start; c <= new_end; c++) { ShenandoahHeapRegion* r = heap->get_region(c); if (c == new_start) { - r->make_humongous_start_bypass(); + r->make_humongous_start_bypass(original_affiliation); } else { - r->make_humongous_cont_bypass(); + r->make_humongous_cont_bypass(original_affiliation); } // Trailing region may be non-full, record the remainder there diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index eb1ab6c3e2c96..7f1e30e44d639 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -64,7 +64,25 @@ class ShenandoahResetBitmapTask : public ShenandoahHeapRegionClosure { void heap_region_do(ShenandoahHeapRegion* region) { if (_heap->is_bitmap_slice_committed(region)) { - _ctx->clear_bitmap(region); + _ctx->clear_bitmap(region); + } + } + + bool is_thread_safe() { return true; } +}; + +class ShenandoahSquirrelAwayCardTable: public ShenandoahHeapRegionClosure { + private: + ShenandoahHeap* _heap; + RememberedScanner* _scanner; + public: + ShenandoahSquirrelAwayCardTable() : + _heap(ShenandoahHeap::heap()), + _scanner(_heap->card_scan()) {} + + void heap_region_do(ShenandoahHeapRegion* region) { + if (region->is_old()) { + _scanner->reset_remset(region->bottom(), ShenandoahHeapRegion::region_size_words()); } } @@ -135,25 +153,54 @@ void ShenandoahGeneration::reset_mark_bitmap() { parallel_heap_region_iterate(&task); } -void ShenandoahGeneration::prepare_gc() { +// The ideal is to swap the remembered set so the safepoint effort is no more than a few pointer manipulations. +// However, limitations in the implementation of the mutator write-barrier make it difficult to simply change the +// location of the card table. So the interim implementation of swap_remembered_set will copy the write-table +// onto the read-table and will then clear the write-table. +void ShenandoahGeneration::swap_remembered_set() { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + heap->assert_gc_workers(heap->workers()->active_workers()); + shenandoah_assert_safepoint(); + + // TODO: Eventually, we want replace this with a constant-time exchange of pointers. + ShenandoahSquirrelAwayCardTable task; + heap->old_generation()->parallel_heap_region_iterate(&task); +} + +void ShenandoahGeneration::prepare_gc(bool do_old_gc_bootstrap) { // Reset mark bitmap for this generation (typically young) reset_mark_bitmap(); + if (do_old_gc_bootstrap) { + // Reset mark bitmap for old regions also. Note that do_old_gc_bootstrap is only true if this generation is YOUNG. + ShenandoahHeap::heap()->old_generation()->reset_mark_bitmap(); + } // Capture Top At Mark Start for this generation (typically young) ShenandoahResetUpdateRegionStateClosure cl; parallel_heap_region_iterate(&cl); + if (do_old_gc_bootstrap) { + // Capture top at mark start for both old-gen regions also. Note that do_old_gc_bootstrap is only true if generation is YOUNG. + ShenandoahHeap::heap()->old_generation()->parallel_heap_region_iterate(&cl); + } } -void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) { +// Returns true iff the chosen collection set includes a mix of young-gen and old-gen regions. +bool ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) { + bool result; ShenandoahHeap* heap = ShenandoahHeap::heap(); assert(!heap->is_full_gc_in_progress(), "Only for concurrent and degenerated GC"); + assert(generation_mode() != OLD, "Only YOUNG and GLOBAL GC perform evacuations"); { ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_update_region_states : ShenandoahPhaseTimings::degen_gc_final_update_region_states); ShenandoahFinalMarkUpdateRegionStateClosure cl(complete_marking_context()); - parallel_heap_region_iterate(&cl); + parallel_heap_region_iterate(&cl); heap->assert_pinned_region_status(); + + // Also capture update_watermark for old-gen regions. + ShenandoahCaptureUpdateWaterMarkForOld old_cl(complete_marking_context()); + heap->old_generation()->parallel_heap_region_iterate(&old_cl); } { @@ -161,7 +208,7 @@ void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) { ShenandoahPhaseTimings::degen_gc_choose_cset); ShenandoahHeapLocker locker(heap->lock()); heap->collection_set()->clear(); - _heuristics->choose_collection_set(heap->collection_set(), heap->old_heuristics()); + result = _heuristics->choose_collection_set(heap->collection_set(), heap->old_heuristics()); } { @@ -170,6 +217,7 @@ void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) { ShenandoahHeapLocker locker(heap->lock()); heap->free_set()->rebuild(); } + return result; } bool ShenandoahGeneration::is_bitmap_clear() { @@ -178,8 +226,9 @@ bool ShenandoahGeneration::is_bitmap_clear() { size_t num_regions = heap->num_regions(); for (size_t idx = 0; idx < num_regions; idx++) { ShenandoahHeapRegion* r = heap->get_region(idx); - if (contains(r)) { - if (heap->is_bitmap_slice_committed(r) && !context->is_bitmap_clear_range(r->bottom(), r->end())) { + if (contains(r) && (r->affiliation() != FREE)) { + if (heap->is_bitmap_slice_committed(r) && (context->top_at_mark_start(r) > r->bottom()) && + !context->is_bitmap_clear_range(r->bottom(), r->end())) { return false; } } @@ -246,14 +295,13 @@ ShenandoahObjToScanQueueSet* ShenandoahGeneration::old_gen_task_queues() const { } void ShenandoahGeneration::scan_remembered_set() { - shenandoah_assert_safepoint(); assert(generation_mode() == YOUNG, "Should only scan remembered set for young generation."); ShenandoahHeap* const heap = ShenandoahHeap::heap(); uint nworkers = heap->workers()->active_workers(); reserve_task_queues(nworkers); - ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_scan_rset); + ShenandoahConcurrentPhase gc_phase("Concurrent remembered set scanning", ShenandoahPhaseTimings::init_scan_rset); ShenandoahReferenceProcessor* rp = heap->ref_processor(); ShenandoahRegionIterator regions; ShenandoahScanRememberedTask task(task_queues(), old_gen_task_queues(), rp, ®ions); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index 4e7a0a3b27c95..4cd218055b944 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -81,9 +81,14 @@ class ShenandoahGeneration : public CHeapObj { // Used directly by FullGC void reset_mark_bitmap(); + // Used by concurrent and degenerated GC to reset remembered set. + void swap_remembered_set(); + // Used by concurrent and degenerated GC to reset regions. - virtual void prepare_gc(); - void prepare_regions_and_collection_set(bool concurrent); + virtual void prepare_gc(bool do_old_gc_bootstrap); + + // Return true iff prepared collection set includes at least one old-gen HeapRegion. + bool prepare_regions_and_collection_set(bool concurrent); // Cancel marking (used by Full collect and when cancelling cycle). void cancel_marking(); @@ -115,6 +120,7 @@ class ShenandoahGeneration : public CHeapObj { virtual void reserve_task_queues(uint workers); virtual ShenandoahObjToScanQueueSet* old_gen_task_queues() const; + // Scan remembered set at start of concurrent young-gen marking. */ void scan_remembered_set(); void increment_affiliated_region_count(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp index 2ec2b36cc9141..11963a9439baa 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp @@ -87,8 +87,8 @@ bool ShenandoahGlobalGeneration::is_concurrent_mark_in_progress() { return heap->is_concurrent_mark_in_progress(); } -void ShenandoahGlobalGeneration::prepare_gc() { - ShenandoahGeneration::prepare_gc(); +void ShenandoahGlobalGeneration::prepare_gc(bool do_old_gc_bootstrap) { + ShenandoahGeneration::prepare_gc(do_old_gc_bootstrap); ShenandoahHeap* heap = ShenandoahHeap::heap(); if (heap->mode()->is_generational()) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp index 30366e3236e62..0f70f3ce04ac9 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp @@ -42,7 +42,7 @@ class ShenandoahGlobalGeneration : public ShenandoahGeneration { virtual size_t used() const; virtual size_t available() const; - virtual void prepare_gc(); + virtual void prepare_gc(bool do_old_gc_bootstrap); virtual void set_concurrent_mark_in_progress(bool in_progress); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 2549461cd23f2..c8e2e93193e2a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -79,6 +79,8 @@ #include "gc/shenandoah/shenandoahJfrSupport.hpp" #endif +#include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp" + #include "classfile/systemDictionary.hpp" #include "memory/classLoaderMetaspace.hpp" #include "oops/compressedOops.inline.hpp" @@ -221,7 +223,8 @@ jint ShenandoahHeap::initialize() { // if (mode()->is_generational()) { ShenandoahDirectCardMarkRememberedSet *rs; - size_t card_count = ShenandoahBarrierSet::barrier_set()->card_table()->cards_required(heap_rs.size() / HeapWordSize) - 1; + ShenandoahCardTable* card_table = ShenandoahBarrierSet::barrier_set()->card_table(); + size_t card_count = card_table->cards_required(heap_rs.size() / HeapWordSize) - 1; rs = new ShenandoahDirectCardMarkRememberedSet(ShenandoahBarrierSet::barrier_set()->card_table(), card_count); _card_scan = new ShenandoahScanRemembered(rs); } @@ -479,7 +482,6 @@ void ShenandoahHeap::initialize_heuristics() { _gc_mode->name())); } - // ojo: want to instantiate a ShenandoahOldHeuristics object here _old_heuristics = _old_generation->initialize_old_heuristics(_gc_mode); _global_generation->initialize_heuristics(_gc_mode); _young_generation->initialize_heuristics(_gc_mode); @@ -494,6 +496,7 @@ ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) : CollectedHeap(), _gc_generation(NULL), _old_heuristics(nullptr), + _mixed_evac(false), _initial_size(0), _used(0), _committed(0), @@ -621,6 +624,10 @@ void ShenandoahHeap::post_initialize() { JFR_ONLY(ShenandoahJFRSupport::register_jfr_type_serializers()); } +bool ShenandoahHeap::doing_mixed_evacuations() { + return (_old_heuristics->unprocessed_old_collection_candidates() > 0); +} + bool ShenandoahHeap::is_gc_generation_young() const { return _gc_generation != NULL && _gc_generation->generation_mode() == YOUNG; } @@ -890,7 +897,7 @@ void ShenandoahHeap::retire_plab(PLAB* plab) { // If retiring the plab created a filler object, then we // need to register it with our card scanner so it can // safely walk the region backing the plab. - card_scan()->register_object(top); + card_scan()->register_object_wo_lock(top); } } @@ -1017,6 +1024,26 @@ HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) { HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req, bool& in_new_region) { ShenandoahHeapLocker locker(lock()); HeapWord* result = _free_set->allocate(req, in_new_region); + if (result != NULL && req.affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) { + // Register the newly allocated object while we're holding the global lock since there's no synchronization + // built in to the implementation of register_object(). There are potential races when multiple independent + // threads are allocating objects, some of which might span the same card region. For example, consider + // a card table's memory region within which three objects are being allocated by three different threads: + // + // objects being "concurrently" allocated: + // [-----a------][-----b-----][--------------c------------------] + // [---- card table memory range --------------] + // + // Before any objects are allocated, this card's memory range holds no objects. Note that: + // allocation of object a wants to set the has-object, first-start, and last-start attributes of the preceding card region. + // allocation of object b wants to set the has-object, first-start, and last-start attributes of this card region. + // allocation of object c also wants to set the has-object, first-start, and last-start attributes of this card region. + // + // The thread allocating b and the thread allocating c can "race" in various ways, resulting in confusion, such as last-start + // representing object b while first-start represents object c. This is why we need to require all register_object() + // invocations to be "mutually exclusive" with respect to each card's memory range. + ShenandoahHeap::heap()->card_scan()->register_object(result); + } return result; } @@ -2105,29 +2132,34 @@ class ShenandoahUpdateHeapRefsTask : public AbstractGangTask { private: ShenandoahHeap* _heap; ShenandoahRegionIterator* _regions; + bool _mixed_evac; // true iff most recent evacuation includes old-gen HeapRegions + public: - ShenandoahUpdateHeapRefsTask(ShenandoahRegionIterator* regions) : + ShenandoahUpdateHeapRefsTask(ShenandoahRegionIterator* regions, bool mixed_evac) : AbstractGangTask("Shenandoah Update References"), _heap(ShenandoahHeap::heap()), - _regions(regions) { + _regions(regions), + _mixed_evac(mixed_evac) + { } void work(uint worker_id) { if (CONCURRENT) { ShenandoahConcurrentWorkerSession worker_session(worker_id); ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers); - do_work(); + do_work(worker_id); } else { ShenandoahParallelWorkerSession worker_session(worker_id); - do_work(); + do_work(worker_id); } } private: template - void do_work() { + void do_work(uint worker_id) { T cl; ShenandoahHeapRegion* r = _regions->next(); + // We update references for global, old, and young collections. assert(_heap->active_generation()->is_mark_complete(), "Expected complete marking"); ShenandoahMarkingContext* const ctx = _heap->marking_context(); @@ -2135,24 +2167,82 @@ class ShenandoahUpdateHeapRefsTask : public AbstractGangTask { while (r != NULL) { HeapWord* update_watermark = r->get_update_watermark(); assert (update_watermark >= r->bottom(), "sanity"); - if (r->is_active() && !r->is_cset()) { - if (!_heap->mode()->is_generational() || r->affiliation() == YOUNG_GENERATION) { + if (!_heap->mode()->is_generational() || (r->affiliation() == ShenandoahRegionAffiliation::YOUNG_GENERATION)) { _heap->marked_object_oop_iterate(r, &cl, update_watermark); - } else if (r->affiliation() == OLD_GENERATION) { - if (!_heap->is_gc_generation_young() || is_mixed) { - // Old region in global or mixed cycle (in which case, old regions should be marked). - // We need to make sure that the next remembered set scan does not iterate over dead objects - // which haven't had their references updated. - r->oop_iterate(&cl, /*fill_dead_objects*/ true, /* reregister_coalesced_objects */ true); + } else if (r->affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) { + if (_heap->active_generation()->generation_mode() == GLOBAL) { + // This code is only relevant to GLOBAL GC. With OLD GC, all coalescing and filling is done before any relevant + // evacuations. + + // This is an old region in a global cycle. Make sure that the next cycle does not iterate over dead objects + // which haven't had their references updated. This is not a promotion. + r->global_oop_iterate_and_fill_dead(&cl); } else { - // Old region in a young cycle with no old regions. - if (!ShenandoahUseSimpleCardScanning) { - _heap->card_scan()->process_region(r, &cl); - } else if (ShenandoahBarrierSet::barrier_set()->card_table()->is_dirty(MemRegion(r->bottom(), r->top()))) { - update_all_references(&cl, r, update_watermark); + // Old region in a young cycle or mixed cycle. + if (ShenandoahUseSimpleCardScanning) { + if (ShenandoahBarrierSet::barrier_set()->card_table()->is_dirty(MemRegion(r->bottom(), r->top()))) { + update_all_references(&cl, r, update_watermark ); + } + } else if (!_mixed_evac) { + // This is a young evac.. + _heap->card_scan()->process_region(r, &cl, true); + } else { + // This is a _mixed_evac. + // + // TODO: For _mixed_evac, consider building an old-gen remembered set that allows restricted updating + // within old-gen HeapRegions. This remembered set can be constructed by old-gen concurrent marking + // and augmented by card marking. For example, old-gen concurrent marking can remember for each old-gen + // card which other old-gen regions it refers to: none, one-other specifically, multiple-other non-specific. + // Update-references when _mixed_evac processess each old-gen memory range that has a traditional DIRTY + // card or if the "old-gen remembered set" indicates that this card holds pointers specifically to an + // old-gen region in the most recent collection set, or if this card holds pointers to other non-specific + // old-gen heap regions. + if (r->is_humongous()) { + r->oop_iterate_humongous(&cl); + } else { + // This is a mixed evacuation. Old regions that are candidates for collection have not been coalesced + // and filled. Use mark bits to find objects that need to be updated. + // + // Future TODO: establish a second remembered set to identify which old-gen regions point to other old-gen + // regions which are in the collection set for a particular mixed evacuation. + HeapWord *p = r->bottom(); + ShenandoahObjectToOopBoundedClosure objs(&cl, p, update_watermark); + + // Anything beyond update_watermark was allocated during evacuation. Thus, it is known to not hold + // references to collection set objects. + while (p < update_watermark) { + oop obj = oop(p); + if (ctx->is_marked(obj)) { + objs.do_object(obj); + p += obj->size(); + } else { + // This object is not marked so we don't scan it. + HeapWord* tams = ctx->top_at_mark_start(r); + if (p >= tams) { + p += obj->size(); + } else { + p = ctx->get_next_marked_addr(p, tams); + } + } + } + } } } + } else { + // Because updating of references runs concurrently, it is possible that a FREE inactive region transitions + // to a non-free active region while this loop is executing. Whenever this happens, the changing of a region's + // active status may propagate at a different speed than the changing of the region's affiliation. + + // When we reach this control point, it is because a race has allowed a region's is_active() status to be seen + // by this thread before the region's affiliation() is seen by this thread. + + // It's ok for this race to occur because the newly transformed region does not have any references to be + // updated. + + assert(r->get_update_watermark() == r->bottom(), + "%s Region " SIZE_FORMAT " is_active but not recognized as YOUNG or OLD so must be newly transitioned from FREE", + affiliation_name(r->affiliation()), r->index()); } } if (ShenandoahPacing) { @@ -2189,10 +2279,10 @@ void ShenandoahHeap::update_heap_references(bool concurrent) { assert(!is_full_gc_in_progress(), "Only for concurrent and degenerated GC"); if (concurrent) { - ShenandoahUpdateHeapRefsTask task(&_update_refs_iterator); + ShenandoahUpdateHeapRefsTask task(&_update_refs_iterator, _mixed_evac); workers()->run_task(&task); } else { - ShenandoahUpdateHeapRefsTask task(&_update_refs_iterator); + ShenandoahUpdateHeapRefsTask task(&_update_refs_iterator, _mixed_evac); workers()->run_task(&task); } } @@ -2441,6 +2531,7 @@ void ShenandoahHeap::flush_liveness_cache(uint worker_id) { assert(worker_id < _max_workers, "sanity"); assert(_liveness_cache != NULL, "sanity"); ShenandoahLiveData* ld = _liveness_cache[worker_id]; + for (uint i = 0; i < num_regions(); i++) { ShenandoahLiveData live = ld[i]; if (live > 0) { @@ -2475,3 +2566,185 @@ template<> void ShenandoahGenerationRegionClosure::heap_region_do(ShenandoahHeapRegion* region) { _cl->heap_region_do(region); } + +// Assure that the remember set has a dirty card everywhere there is an interesting pointer. +// This examines the read_card_table between bottom() and top() since all PLABS are retired +// before the safepoint for init_mark. Actually, we retire them before update-references and don't +// restore them until the start of evacuation. +void ShenandoahHeap::verify_rem_set_at_mark() { + shenandoah_assert_safepoint(); + assert(mode()->is_generational(), "Only verify remembered set for generational operational modes"); + + ShenandoahRegionIterator iterator; + ShenandoahMarkingContext* mark_context = marking_context(); + RememberedScanner* scanner = card_scan(); + ShenandoahVerifyRemSetClosure check_interesting_pointers(true); + ShenandoahMarkingContext* ctx; + + if (doing_mixed_evacuations()) { + ctx = mark_context; + } else { + ctx = nullptr; + } + + while (iterator.has_next()) { + ShenandoahHeapRegion* r = iterator.next(); + if (r == nullptr) + break; + if (r->is_old()) { + HeapWord* obj_addr = r->bottom(); + if (r->is_humongous_start()) { + oop obj = oop(obj_addr); + if (!ctx || ctx->is_marked(obj)) { + // For humongous objects, the typical object is an array, so the following checks may be overkill + // For regular objects (not object arrays), if the card holding the start of the object is dirty, + // we do not need to verify that cards spanning interesting pointers within this object are dirty. + if (!scanner->is_card_dirty(obj_addr) || obj->is_objArray()) { + obj->oop_iterate(&check_interesting_pointers); + } + // else, object's start is marked dirty and obj is not an objArray, so any interesting pointers are covered + } + // else, this humongous object is not marked so no need to verify its internal pointers + if (!scanner->verify_registration(obj_addr, obj->size())) { + ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, obj_addr, NULL, + "Verify init-mark remembered set violation", "object not properly registered", __FILE__, __LINE__); + } + } else if (!r->is_humongous()) { + HeapWord* t = r->top(); + while (obj_addr < t) { + oop obj = oop(obj_addr); + // ctx->is_marked() returns true if mark bit set or if obj above TAMS. + if (!ctx || ctx->is_marked(obj)) { + // For regular objects (not object arrays), if the card holding the start of the object is dirty, + // we do not need to verify that cards spanning interesting pointers within this object are dirty. + if (!scanner->is_card_dirty(obj_addr) || obj->is_objArray()) { + obj->oop_iterate(&check_interesting_pointers); + } + // else, object's start is marked dirty and obj is not an objArray, so any interesting pointers are covered + if (!scanner->verify_registration(obj_addr, obj->size())) { + ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, obj_addr, NULL, + "Verify init-mark remembered set violation", "object not properly registered", __FILE__, __LINE__); + } + obj_addr += obj->size(); + } else { + // This object is not live so we don't verify dirty cards contained therein + ShenandoahHeapRegion* r = heap_region_containing(obj_addr); + HeapWord* tams = ctx->top_at_mark_start(r); + if (obj_addr >= tams) { + obj_addr += obj->size(); + } else { + obj_addr = ctx->get_next_marked_addr(obj_addr, tams); + } + } + } + } // else, we ignore humongous continuation region + } // else, this is not an OLD region so we ignore it + } // all regions have been processed +} + +// Assure that the remember set has a dirty card everywhere there is an interesting pointer. Even though +// the update-references scan of remembered set only examines cards up to update_watermark, the remembered +// set should be valid through top. This examines the write_card_table between bottom() and top() because +// all PLABS are retired immediately before the start of update refs. +void ShenandoahHeap::verify_rem_set_at_update_ref() { + shenandoah_assert_safepoint(); + assert(mode()->is_generational(), "Only verify remembered set for generational operational modes"); + + ShenandoahRegionIterator iterator; + ShenandoahMarkingContext* mark_context = marking_context(); + RememberedScanner* scanner = card_scan(); + ShenandoahVerifyRemSetClosure check_interesting_pointers(false); + ShenandoahMarkingContext* ctx; + + if (doing_mixed_evacuations()) { + ctx = mark_context; + } else { + ctx = nullptr; + } + + while (iterator.has_next()) { + ShenandoahHeapRegion* r = iterator.next(); + if (r == nullptr) + break; + if (r->is_old() && !r->is_cset()) { + HeapWord* obj_addr = r->bottom(); + if (r->is_humongous_start()) { + oop obj = oop(obj_addr); + if (!ctx || ctx->is_marked(obj)) { + size_t card_index = scanner->card_index_for_addr(obj_addr); + // For humongous objects, the typical object is an array, so the following checks may be overkill + // For regular objects (not object arrays), if the card holding the start of the object is dirty, + // we do not need to verify that cards spanning interesting pointers within this object are dirty. + if (!scanner->is_write_card_dirty(card_index) || obj->is_objArray()) { + obj->oop_iterate(&check_interesting_pointers); + } + // else, object's start is marked dirty and obj is not an objArray, so any interesting pointers are covered + } + // else, this humongous object is not live so no need to verify its internal pointers + if (!scanner->verify_registration(obj_addr, obj->size())) { + ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, obj_addr, NULL, + "Verify init-update-references remembered set violation", "object not properly registered", __FILE__, __LINE__); + } + } else if (!r->is_humongous()) { + HeapWord* t = r->get_update_watermark(); + while (obj_addr < t) { + oop obj = oop(obj_addr); + // ctx->is_marked() returns true if mark bit set or if obj above TAMS. + if (!ctx || ctx->is_marked(obj)) { + size_t card_index = scanner->card_index_for_addr(obj_addr); + // For regular objects (not object arrays), if the card holding the start of the object is dirty, + // we do not need to verify that cards spanning interesting pointers within this object are dirty. + if (!scanner->is_write_card_dirty(card_index) || obj->is_objArray()) { + obj->oop_iterate(&check_interesting_pointers); + } + // else, object's start is marked dirty and obj is not an objArray, so any interesting pointers are covered + if (!scanner->verify_registration(obj_addr, obj->size())) { + ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, obj_addr, NULL, + "Verify init-update-references remembered set violation", "object not properly registered", __FILE__, __LINE__); + } + obj_addr += obj->size(); + } else { + // This object is not live so we don't verify dirty cards contained therein + ShenandoahHeapRegion* r = heap_region_containing(obj_addr); + HeapWord* tams = ctx->top_at_mark_start(r); + if (obj_addr >= tams) { + obj_addr += obj->size(); + } else { + obj_addr = ctx->get_next_marked_addr(obj_addr, tams); + } + } + } + // Update references only cares about remembered set below update_watermark, but entire remset should be valid + // We're at safepoint and all LABs have been flushed, so we can parse all the way to top(). + t = r->top(); + while (obj_addr < t) { + oop obj = oop(obj_addr); + // ctx->is_marked() returns true if mark bit set or if obj above TAMS. + if (!ctx || ctx->is_marked(obj)) { + size_t card_index = scanner->card_index_for_addr(obj_addr); + // For regular objects (not object arrays), if the card holding the start of the object is dirty, + // we do not need to verify that cards spanning interesting pointers within this object are dirty. + if (!scanner->is_write_card_dirty(card_index) || obj->is_objArray()) { + obj->oop_iterate(&check_interesting_pointers); + } + // else, object's start is marked dirty and obj is not an objArray, so any interesting pointers are covered + if (!scanner->verify_registration(obj_addr, obj->size())) { + ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, obj_addr, NULL, + "Verify init-update-references remembered set violation", "object not properly registered", __FILE__, __LINE__); + } + obj_addr += obj->size(); + } else { + // This object is not live so we don't verify dirty cards contained therein + ShenandoahHeapRegion* r = heap_region_containing(obj_addr); + HeapWord* tams = ctx->top_at_mark_start(r); + if (obj_addr >= tams) { + obj_addr += obj->size(); + } else { + obj_addr = ctx->get_next_marked_addr(obj_addr, tams); + } + } + } + } + } // else, we don't care about this region + } +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index 8cfb99c561245..82b7a0acf9073 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -149,6 +149,7 @@ class ShenandoahHeap : public CollectedHeap { ShenandoahHeapLock _lock; ShenandoahGeneration* _gc_generation; ShenandoahOldHeuristics* _old_heuristics; + bool _mixed_evac; // true iff most recent evac included at least one old-gen HeapRegion public: ShenandoahHeapLock* lock() { @@ -164,10 +165,16 @@ class ShenandoahHeap : public CollectedHeap { _gc_generation = generation; } + void set_mixed_evac(bool mixed_evac) { + _mixed_evac = mixed_evac; + } + ShenandoahOldHeuristics* old_heuristics() { return _old_heuristics; } + bool doing_mixed_evacuations(); + bool is_gc_generation_young() const; // ---------- Initialization, termination, identification, printing routines @@ -195,6 +202,8 @@ class ShenandoahHeap : public CollectedHeap { void prepare_for_verify(); void verify(VerifyOption vo); + void verify_rem_set_at_mark(); + void verify_rem_set_at_update_ref(); // ---------- Heap counters and metrics // @@ -371,6 +380,11 @@ class ShenandoahHeap : public CollectedHeap { double _cancel_requested_time; ShenandoahSharedEnumFlag _cancelled_gc; + + // Returns true if cancel request was successfully communicated. + // Returns false if some other thread already communicated cancel + // request. A true return value does not mean GC has been + // cancelled, only that the process of cancelling GC has begun. bool try_cancel_gc(); public: diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp index c22cffeb8be74..95c00e15de40b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp @@ -230,6 +230,9 @@ inline HeapWord* ShenandoahHeap::allocate_from_plab(Thread* thread, size_t size) obj = allocate_from_plab_slow(thread, size); } + if (mode()->is_generational() && obj != NULL) { + ShenandoahHeap::heap()->card_scan()->register_object_wo_lock(obj); + } return obj; } @@ -266,6 +269,8 @@ inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) { return try_evacuate_object(p, thread, r, target_gen); } +// try_evacuate_object registers the object and dirties the associated remembered set information when evacuating +// to OLD_GENERATION. inline oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region, ShenandoahRegionAffiliation target_gen) { bool alloc_from_lab = true; HeapWord* copy = NULL; @@ -338,8 +343,16 @@ inline oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, Shenandoah oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val); if (result == copy_val) { if (target_gen == OLD_GENERATION) { - ShenandoahBarrierSet::barrier_set()->card_table()->dirty_MemRegion(MemRegion(copy, size)); - card_scan()->register_object(copy); + if (alloc_from_lab) { + card_scan()->register_object_wo_lock(copy); + } + // else, allocate_memory_under_lock() has already registered the object + + // Mark the entire range of the evacuated object as dirty. At next remembered set scan, + // we will clear dirty bits that do not hold interesting pointers. It's more efficient to + // do this in batch, in a background GC thread than to try to carefully dirty only cards + // that hold interesting pointers right now. + card_scan()->mark_range_as_dirty(copy, size); } // Successfully evacuated. Our copy is now the public one! shenandoah_assert_correct(NULL, copy_val); @@ -374,9 +387,7 @@ inline oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, Shenandoah // we have to keep the fwdptr initialized and pointing to our (stale) copy. fill_with_object(copy, size); shenandoah_assert_correct(NULL, copy_val); - if (target_gen == OLD_GENERATION) { - card_scan()->register_object(copy); - } + // For non-LAB allocations, the object has already been registered } shenandoah_assert_correct(NULL, result); return result; @@ -644,5 +655,4 @@ inline void ShenandoahHeap::mark_card_as_dirty(void* location) { } } - #endif // SHARE_GC_SHENANDOAH_SHENANDOAHHEAP_INLINE_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index b2a5f05cc11be..857f9e189aadf 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -155,9 +155,10 @@ void ShenandoahHeapRegion::make_humongous_start() { } } -void ShenandoahHeapRegion::make_humongous_start_bypass() { +void ShenandoahHeapRegion::make_humongous_start_bypass(ShenandoahRegionAffiliation affiliation) { shenandoah_assert_heaplocked(); assert (ShenandoahHeap::heap()->is_full_gc_in_progress(), "only for full GC"); + set_affiliation(affiliation); reset_age(); switch (_state) { case _empty_committed: @@ -185,9 +186,10 @@ void ShenandoahHeapRegion::make_humongous_cont() { } } -void ShenandoahHeapRegion::make_humongous_cont_bypass() { +void ShenandoahHeapRegion::make_humongous_cont_bypass(ShenandoahRegionAffiliation affiliation) { shenandoah_assert_heaplocked(); assert (ShenandoahHeap::heap()->is_full_gc_in_progress(), "only for full GC"); + set_affiliation(affiliation); reset_age(); switch (_state) { case _empty_committed: @@ -283,7 +285,9 @@ void ShenandoahHeapRegion::make_trash_immediate() { // On this path, we know there are no marked objects in the region, // tell marking context about it to bypass bitmap resets. assert(ShenandoahHeap::heap()->active_generation()->is_mark_complete(), "Marking should be complete here."); - ShenandoahHeap::heap()->marking_context()->reset_top_bitmap(this); + // Leave top_bitmap alone. If it is greater than bottom(), then we still need to clear between bottom() and top_bitmap() + // when this FREE region is repurposed for YOUNG or OLD. + // ShenandoahHeap::heap()->marking_context()->reset_top_bitmap(this); } void ShenandoahHeapRegion::make_empty() { @@ -424,16 +428,22 @@ void ShenandoahHeapRegion::print_on(outputStream* st) const { // oop_iterate without closure void ShenandoahHeapRegion::oop_fill_and_coalesce() { HeapWord* obj_addr = bottom(); - HeapWord* t = top(); assert(!is_humongous(), "No need to fill or coalesce humongous regions"); if (!is_active()) return; ShenandoahHeap* heap = ShenandoahHeap::heap(); ShenandoahMarkingContext* marking_context = heap->marking_context(); + // All objects above TAMS are considered live even though their mark bits will not be set. Note that young- + // gen evacuations that interrupt a long-running old-gen concurrent mark may promote objects into old-gen + // while the old-gen concurrent marking is ongoing. These newly promoted objects will reside above TAMS + // and will be treated as live during the current old-gen marking pass, even though they will not be + // explicitly marked. + HeapWord* t = marking_context->top_at_mark_start(this); // Expect this to be invoked only from within threads perfoming old-gen GC, and expect // old-gen marking to be completed before these threads invoke this service. + assert(heap->active_generation()->generation_mode() == OLD, "sanity"); assert(heap->active_generation()->is_mark_complete(), "sanity"); while (obj_addr < t) { @@ -453,61 +463,107 @@ void ShenandoahHeapRegion::oop_fill_and_coalesce() { } } - -void ShenandoahHeapRegion::oop_iterate(OopIterateClosure* blk, bool fill_dead_objects, bool reregister_coalesced_objects) { +void ShenandoahHeapRegion::global_oop_iterate_and_fill_dead(OopIterateClosure* blk) { if (!is_active()) return; if (is_humongous()) { - // TODO: This doesn't look right. This registers objects if !reregister, and it isn't filling if fill_dead_objects. - // Furthermore, register and fill should be done after iterating. - if (fill_dead_objects && !reregister_coalesced_objects) { - ShenandoahHeap::heap()->card_scan()->register_object(bottom()); - } + // No need to fill dead within humongous regions. Either the entire region is dead, or the entire region is + // unchanged. A humongous region holds no more than one humongous object. oop_iterate_humongous(blk); } else { - oop_iterate_objects(blk, fill_dead_objects, reregister_coalesced_objects); + global_oop_iterate_objects_and_fill_dead(blk); } } -void ShenandoahHeapRegion::oop_iterate_objects(OopIterateClosure* blk, bool fill_dead_objects, bool reregister_coalesced_objects) { +void ShenandoahHeapRegion::global_oop_iterate_objects_and_fill_dead(OopIterateClosure* blk) { assert(!is_humongous(), "no humongous region here"); HeapWord* obj_addr = bottom(); - HeapWord* t = top(); - if (!fill_dead_objects) { - while (obj_addr < t) { - oop obj = oop(obj_addr); + ShenandoahHeap* heap = ShenandoahHeap::heap(); + ShenandoahMarkingContext* marking_context = heap->marking_context(); + RememberedScanner* rem_set_scanner = heap->card_scan(); + // Objects allocated above TAMS are not marked, but are considered live for purposes of current GC efforts. + HeapWord* t = marking_context->top_at_mark_start(this); + + assert(heap->active_generation()->is_mark_complete(), "sanity"); + + while (obj_addr < t) { + oop obj = oop(obj_addr); + if (marking_context->is_marked(obj)) { assert(obj->klass() != NULL, "klass should not be NULL"); + // when promoting an entire region, we have to register the marked objects as well obj_addr += obj->oop_iterate_size(blk); + } else { + // Object is not marked. Coalesce and fill dead object with dead neighbors. + HeapWord* next_marked_obj = marking_context->get_next_marked_addr(obj_addr, t); + assert(next_marked_obj <= t, "next marked object cannot exceed top"); + size_t fill_size = next_marked_obj - obj_addr; + ShenandoahHeap::fill_with_object(obj_addr, fill_size); + + // coalesce_objects() unregisters all but first object subsumed within coalesced range. + rem_set_scanner->coalesce_objects(obj_addr, fill_size); + obj_addr = next_marked_obj; } - } else { - ShenandoahHeap* heap = ShenandoahHeap::heap(); - ShenandoahMarkingContext* marking_context = heap->marking_context(); - assert(heap->active_generation()->is_mark_complete(), "sanity"); - HeapWord* tams = marking_context->top_at_mark_start(this); - - while (obj_addr < t) { - oop obj = oop(obj_addr); - if (marking_context->is_marked(obj)) { - assert(obj->klass() != NULL, "klass should not be NULL"); - if (!reregister_coalesced_objects) { - heap->card_scan()->register_object(obj_addr); - } - obj_addr += obj->oop_iterate_size(blk); - } else { - // Object is not marked. Coalesce and fill dead object with dead neighbors. - HeapWord* next_marked_obj = marking_context->get_next_marked_addr(obj_addr, tams); - assert(next_marked_obj <= tams, "next marked object cannot exceed top at mark start"); - size_t fill_size = next_marked_obj - obj_addr; - ShenandoahHeap::fill_with_object(obj_addr, fill_size); - if (reregister_coalesced_objects) { - heap->card_scan()->coalesce_objects(obj_addr, fill_size); - } else { // establish new crossing map information - heap->card_scan()->register_object(obj_addr); - } - obj_addr = next_marked_obj; - } + } + + // Any object above TAMS and below top() is considered live. + t = top(); + while (obj_addr < t) { + oop obj = oop(obj_addr); + obj_addr += obj->oop_iterate_size(blk); + } +} + +// This function does not set card dirty bits. The decision of which cards to dirty is best +// made in the caller's context. +void ShenandoahHeapRegion::fill_dead_and_register_for_promotion() { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + ShenandoahMarkingContext* marking_context = heap->marking_context(); + HeapWord* obj_addr = bottom(); + RememberedScanner* rem_set_scanner = heap->card_scan(); + // Objects allocated above TAMS are not marked, but are considered live for purposes of current GC efforts. + HeapWord* t = marking_context->top_at_mark_start(this); + + assert(!is_humongous(), "no humongous region here"); + assert(heap->active_generation()->is_mark_complete(), "sanity"); + + // end() might be overkill as end of range, but top() may not align with card boundary. + rem_set_scanner->reset_object_range(bottom(), end()); + while (obj_addr < t) { + oop obj = oop(obj_addr); + if (marking_context->is_marked(obj)) { + assert(obj->klass() != NULL, "klass should not be NULL"); + // when promoting an entire region, we have to register the marked objects as well + rem_set_scanner->register_object_wo_lock(obj_addr); + obj_addr += obj->size(); + } else { + // Object is not marked. Coalesce and fill dead object with dead neighbors. + HeapWord* next_marked_obj = marking_context->get_next_marked_addr(obj_addr, t); + assert(next_marked_obj <= t, "next marked object cannot exceed top"); + size_t fill_size = next_marked_obj - obj_addr; + assert(fill_size >= (size_t) oopDesc::header_size(), + "fill size " SIZE_FORMAT " for obj @ " PTR_FORMAT ", next_marked: " PTR_FORMAT ", TAMS: " PTR_FORMAT " is too small", + fill_size, p2i(obj_addr), p2i(next_marked_obj), p2i(t)); + ShenandoahHeap::fill_with_object(obj_addr, fill_size); + rem_set_scanner->register_object_wo_lock(obj_addr); + obj_addr = next_marked_obj; } } + + // Any object above TAMS and below top() is considered live. + t = top(); + while (obj_addr < t) { + oop obj = oop(obj_addr); + assert(obj->klass() != NULL, "klass should not be NULL"); + // when promoting an entire region, we have to register the marked objects as well + rem_set_scanner->register_object_wo_lock(obj_addr); + obj_addr += obj->size(); + } + + // In case top() does not align with a card boundary, it's necessary to fill remainder of memory beyond top(). + if (top() < end()) { + ShenandoahHeap::fill_with_object(top(), end() - top());; + rem_set_scanner->register_object_wo_lock(obj_addr); + } } void ShenandoahHeapRegion::oop_iterate_humongous(OopIterateClosure* blk) { @@ -789,6 +845,27 @@ size_t ShenandoahHeapRegion::pin_count() const { void ShenandoahHeapRegion::set_affiliation(ShenandoahRegionAffiliation new_affiliation) { ShenandoahHeap* heap = ShenandoahHeap::heap(); + + { + ShenandoahMarkingContext* const ctx = heap->complete_marking_context(); + log_debug(gc)("Setting affiliation of Region " SIZE_FORMAT " from %s to %s, top: " PTR_FORMAT ", TAMS: " PTR_FORMAT + ", watermark: " PTR_FORMAT ", top_bitmap: " PTR_FORMAT "\n", + index(), affiliation_name(_affiliation), affiliation_name(new_affiliation), + p2i(top()), p2i(ctx->top_at_mark_start(this)), p2i(this->get_update_watermark()), p2i(ctx->top_bitmap(this))); + } + +#ifdef ASSERT + { + ShenandoahMarkingContext* const ctx = heap->complete_marking_context(); + size_t idx = this->index(); + HeapWord* top_bitmap = ctx->top_bitmap(this); + + assert(ctx->is_bitmap_clear_range(top_bitmap, _end), + "Region " SIZE_FORMAT ", bitmap should be clear between top_bitmap: " PTR_FORMAT " and end: " PTR_FORMAT, idx, + p2i(top_bitmap), p2i(_end)); + } +#endif + if (_affiliation == new_affiliation) { return; } @@ -807,11 +884,9 @@ void ShenandoahHeapRegion::set_affiliation(ShenandoahRegionAffiliation new_affil heap->old_generation()->decrement_affiliated_region_count(); } - CardTable* card_table = ShenandoahBarrierSet::barrier_set()->card_table(); switch (new_affiliation) { case FREE: assert(!has_live(), "Free region should not have live data"); - card_table->clear_MemRegion(MemRegion(_bottom, _end)); break; case YOUNG_GENERATION: reset_age(); @@ -827,79 +902,94 @@ void ShenandoahHeapRegion::set_affiliation(ShenandoahRegionAffiliation new_affil _affiliation = new_affiliation; } -class UpdateCardValuesClosure : public BasicOopIterateClosure { -private: - void update_card_value(void* address, oop obj) { - if (ShenandoahHeap::heap()->is_in_young(obj)) { - ShenandoahHeap::heap()->mark_card_as_dirty(address); - } - } - -public: - void do_oop(oop* p) { - oop obj = *p; - if (obj != NULL) { - update_card_value(p, obj); - } - } - - void do_oop(narrowOop* p) { - narrowOop o = RawAccess<>::oop_load(p); - if (!CompressedOops::is_null(o)) { - oop obj = CompressedOops::decode_not_null(o); - assert(oopDesc::is_oop(obj), "must be a valid oop"); - update_card_value(p, obj); - } - } -}; - -size_t ShenandoahHeapRegion::promote() { +size_t ShenandoahHeapRegion::promote(bool promoting_all) { + // TODO: Not sure why region promotion must be performed at safepoint. Reconsider this requirement. assert(SafepointSynchronize::is_at_safepoint(), "must be at a safepoint"); + // Note that region promotion occurs at a safepoint following all evacuation. When a region is promoted, we leave + // its TAMS and update_watermark information as is. + // + // Note that update_watermark represents the state of this region as of the moment at which the most recent evacuation + // began. The value of update_watermark is the same for old regions and young regions, as both participate equally in + // the processes of a mixed evacuation. + // + // The meaning of TAMS is different for young-gen and old-gen regions. For a young-gen region, TAMS represents + // top() at start of most recent young-gen concurrent mark. For an old-gen region, TAMS represents top() at start + // of most recent old-gen concurrent mark(). In the case that a young-gen heap region is promoted into old-gen, + // we can preserve its TAMS information with the following understandings: + // 1. The most recent young-GC concurrent mark phase began at the same time or after the most recent old-GC + // concurrent mark phase. + // 2. After the region is promoted, it is still the case that any object within the region that is beneath TAMS + // and is considered alive for the current old GC pass will be "marked" within the current marking context, and + // any object within the region that is above TAMS will be considered alive for the current old GC pass. Objects + // that were dead at promotion time will all reside below TAMS and will be unmarked. ShenandoahHeap* heap = ShenandoahHeap::heap(); ShenandoahMarkingContext* marking_context = heap->marking_context(); assert(heap->active_generation()->is_mark_complete(), "sanity"); assert(affiliation() == YOUNG_GENERATION, "Only young regions can be promoted"); - UpdateCardValuesClosure update_card_values; ShenandoahGeneration* old_generation = heap->old_generation(); ShenandoahGeneration* young_generation = heap->young_generation(); if (is_humongous_start()) { oop obj = oop(bottom()); assert(marking_context->is_marked(obj), "promoted humongous object should be alive"); - heap->card_scan()->register_object(bottom()); + + // Since the humongous region holds only one object, no lock is necessary for this register_object() invocation. + heap->card_scan()->register_object_wo_lock(bottom()); size_t index_limit = index() + ShenandoahHeapRegion::required_regions(obj->size() * HeapWordSize); + + // For this region and each humongous continuation region spanned by this humongous object, change + // affiliation to OLD_GENERATION and adjust the generation-use tallies. The remnant of memory + // in the last humongous region that is not spanned by obj is currently not used. for (size_t i = index(); i < index_limit; i++) { ShenandoahHeapRegion* r = heap->get_region(i); log_debug(gc)("promoting region " SIZE_FORMAT ", from " SIZE_FORMAT " to " SIZE_FORMAT, r->index(), (size_t) r->bottom(), (size_t) r->top()); if (r->top() < r->end()) { ShenandoahHeap::fill_with_object(r->top(), (r->end() - r->top()) / HeapWordSize); - heap->card_scan()->register_object(r->top()); - heap->clear_cards(r->top(), r->end()); + heap->card_scan()->register_object_wo_lock(r->top()); + heap->card_scan()->mark_range_as_clean(top(), r->end() - r->top()); } - heap->dirty_cards(r->bottom(), r->top()); + // We mark the entire humongous object's range as dirty after loop terminates, so no need to dirty the range here r->set_affiliation(OLD_GENERATION); + log_debug(gc)("promoting humongous region " SIZE_FORMAT ", dirtying cards from " SIZE_FORMAT " to " SIZE_FORMAT, + i, (size_t) r->bottom(), (size_t) r->top()); old_generation->increase_used(r->used()); young_generation->decrease_used(r->used()); } + if (promoting_all || obj->is_typeArray()) { + // Primitive arrays don't need to be scanned. Likewise, if we are promoting_all, there's nothing + // left in young-gen, so there can exist no "interesting" pointers. See above TODO question about requiring + // region promotion at safepoint. If we're not at a safepoint, then we can't really "promote all" without + // directing new allocations to old-gen. That's probably not what we want. The whole "promote-all strategy" + // probably needs to be revisited at some future point. + heap->card_scan()->mark_range_as_clean(bottom(), obj->size()); + } else { + heap->card_scan()->mark_range_as_dirty(bottom(), obj->size()); + } return index_limit - index(); } else { - log_debug(gc)("promoting region " SIZE_FORMAT ", from " SIZE_FORMAT " to " SIZE_FORMAT, + log_debug(gc)("promoting region " SIZE_FORMAT ", dirtying cards from " SIZE_FORMAT " to " SIZE_FORMAT, index(), (size_t) bottom(), (size_t) top()); assert(!is_humongous_continuation(), "should not promote humongous object continuation in isolation"); + fill_dead_and_register_for_promotion(); + // Rather than scanning entire contents of the promoted region right now to determine which + // cards to mark as dirty, we just mark them all as dirty (unless promoting_all). Later, when we + // scan the remembered set, we will clear cards that are found to not contain live references to + // young memory. Ultimately, this approach is more efficient as it only scans the "dirty" cards + // once and the clean cards once. The alternative approach of scanning all cards now and then + // scanning dirty cards again at next concurrent mark pass scans the clean cards once and the dirty + // cards twice. + if (promoting_all) { + heap->card_scan()->mark_range_as_clean(bottom(), top() - bottom()); + } else { + heap->card_scan()->mark_range_as_dirty(bottom(), top() - bottom()); + } set_affiliation(OLD_GENERATION); old_generation->increase_used(used()); young_generation->decrease_used(used()); - - // In terms of card marking, We could just set the whole occupied range in this region to dirty instead of iterating here. - // Card scanning could correct false positives later and that would be more efficient. - // But oop_iterate_objects() has other, indispensable effects: filling dead objects and registering object starts. - // So while we are already doing this here, we may as well also set more precise card values. - heap->dirty_cards(bottom(), top()); - oop_iterate_objects(&update_card_values, /*fill_dead_objects*/ true, /* reregister_coalesced_objects */ false); return 1; } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp index 7ae228de8206a..beaf5f4f275b3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp @@ -172,8 +172,8 @@ class ShenandoahHeapRegion { void make_regular_bypass(); void make_humongous_start(); void make_humongous_cont(); - void make_humongous_start_bypass(); - void make_humongous_cont_bypass(); + void make_humongous_start_bypass(ShenandoahRegionAffiliation affiliation); + void make_humongous_cont_bypass(ShenandoahRegionAffiliation affiliation); void make_pinned(); void make_unpinned(); void make_cset(); @@ -367,10 +367,14 @@ class ShenandoahHeapRegion { void recycle(); - // coalesce contiguous spans of garbage objects by filling header and reregistering start locations with remembered set. + // Coalesce contiguous spans of garbage objects by filling header and reregistering start locations with remembered set. + // This is used by old-gen GC following concurrent marking to make old-gen HeapRegions parseable. void oop_fill_and_coalesce(); - void oop_iterate(OopIterateClosure* cl, bool fill_dead_objects = false, bool reregister_coalesced_objects = false); + // During global collections, this service iterates through an old-gen heap region that is not part of collection + // set to fill and register ranges of dead memory. Note that live objects were previously registered. Some dead objects + // that are subsumed into coalesced ranges of dead memory need to be "unregistered". + void global_oop_iterate_and_fill_dead(OopIterateClosure* cl); void oop_iterate_humongous(OopIterateClosure* cl); HeapWord* block_start(const void* p) const; @@ -414,15 +418,24 @@ class ShenandoahHeapRegion { void increment_age() { if (_age < markWord::max_age) { _age++; } } void reset_age() { _age = 0; } - // If this is a humongous start, returns the number of regions in the object. - size_t promote(); + // Adjusts remembered set information by setting all cards to clean if promoting all, setting + // all cards to dirty otherwise. + // + // Returns the number of regions promoted, which is generally one, but may be greater than 1 if + // this is humongous region with multiple continuations. + size_t promote(bool promoting_all); private: void do_commit(); void do_uncommit(); - void oop_iterate_objects(OopIterateClosure* cl, bool fill_dead_objects, bool reregister_coalesced_objects); - void oop_iterate_objects(bool fill_dead_objects, bool reregister_coalesced_objects); + // This is an old-region that was not part of the collection set during a GLOBAL collection. We coalesce the dead + // objects, but do not need to register the live objects as they are already registered. + void global_oop_iterate_objects_and_fill_dead(OopIterateClosure* cl); + + // Process the contents of a region when it is being promoted en masse by registering each marked object, coalescing + // contiguous ranges of unmarked objects into registered dead objects. Do not touch card marks. + void fill_dead_and_register_for_promotion(); inline void internal_increase_live_data(size_t s); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp index 316a1c0277e34..12e129ab14df9 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp @@ -88,11 +88,13 @@ inline void ShenandoahHeapRegion::internal_increase_live_data(size_t s) { size_t live_bytes = new_live_data * HeapWordSize; size_t used_bytes = used(); assert(live_bytes <= used_bytes, - "can't have more live data than used: " SIZE_FORMAT ", " SIZE_FORMAT, live_bytes, used_bytes); + "%s Region " SIZE_FORMAT " can't have more live data than used: " SIZE_FORMAT ", " SIZE_FORMAT " after adding " SIZE_FORMAT, + affiliation_name(affiliation()), index(), live_bytes, used_bytes, s * HeapWordSize); #endif } inline void ShenandoahHeapRegion::clear_live_data() { + log_debug(gc)("SHR::clear_live_data on %s Region " SIZE_FORMAT, affiliation_name(affiliation()), index()); Atomic::store(&_live_data, (size_t)0); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp index e6300b00e79cc..266227606d6c6 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp @@ -178,7 +178,7 @@ void ShenandoahMark::mark_loop_work(T* cl, ShenandoahLiveData* live_data, uint w } void ShenandoahMark::mark_loop(GenerationMode generation, uint worker_id, TaskTerminator* terminator, ShenandoahReferenceProcessor *rp, - bool cancellable, bool strdedup) { + bool cancellable, bool strdedup) { bool update_refs = ShenandoahHeap::heap()->has_forwarded_objects(); switch (generation) { case YOUNG: { @@ -212,17 +212,3 @@ void ShenandoahMark::mark_loop(GenerationMode generation, uint worker_id, TaskTe } } -template<> -bool ShenandoahMark::in_generation(oop obj) { - return ShenandoahHeap::heap()->is_in_young(obj); -} - -template<> -bool ShenandoahMark::in_generation(oop obj) { - return ShenandoahHeap::heap()->is_in_old(obj); -} - -template<> -bool ShenandoahMark::in_generation(oop obj) { - return true; -} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.hpp index 4ee8952052604..743fa5c10ef4b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMark.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.hpp @@ -71,6 +71,8 @@ class ShenandoahMark: public StackObj { inline ShenandoahObjToScanQueue* get_queue(uint index) const; inline ShenandoahObjToScanQueue* get_old_queue(uint index) const; + inline ShenandoahGeneration* generation() { return _generation; }; + // ---------- Marking loop and tasks private: template diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp index a3532a1bf048c..059ed8ed52c6c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp @@ -103,6 +103,7 @@ inline void ShenandoahMark::count_liveness(ShenandoahLiveData* live_data, oop ob if (!region->is_humongous_start()) { assert(!region->is_humongous(), "Cannot have continuations here"); + assert(region->affiliation() != FREE, "Do not count live data within Free Regular Region " SIZE_FORMAT, region_idx); ShenandoahLiveData cur = live_data[region_idx]; size_t new_val = size + cur; if (new_val >= SHENANDOAH_LIVEDATA_MAX) { @@ -117,9 +118,11 @@ inline void ShenandoahMark::count_liveness(ShenandoahLiveData* live_data, oop ob shenandoah_assert_in_correct_region(NULL, obj); size_t num_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize); + assert(region->affiliation() != FREE, "Do not count live data within FREE Humongous Start Region " SIZE_FORMAT, region_idx); for (size_t i = region_idx; i < region_idx + num_regions; i++) { ShenandoahHeapRegion* chain_reg = heap->get_region(i); assert(chain_reg->is_humongous(), "Expecting a humongous region"); + assert(chain_reg->affiliation() != FREE, "Do not count live data within FREE Humongous Continuation Region " SIZE_FORMAT, i); chain_reg->increase_live_data_gc_words(chain_reg->used() >> LogHeapWordSize); } } @@ -256,6 +259,19 @@ class ShenandoahSATBBufferClosure : public SATBBufferClosure { } }; +template +bool ShenandoahMark::in_generation(oop obj) { + // Each in-line expansion of in_generation() resolves GENERATION at compile time. + if (GENERATION == YOUNG) + return ShenandoahHeap::heap()->is_in_young(obj); + else if (GENERATION == OLD) + return ShenandoahHeap::heap()->is_in_old(obj); + else if (GENERATION == GLOBAL) + return true; + else + return false; +} + template inline void ShenandoahMark::mark_through_ref(T *p, ShenandoahObjToScanQueue* q, ShenandoahObjToScanQueue* old, ShenandoahMarkingContext* const mark_context, bool weak) { T o = RawAccess<>::oop_load(p); @@ -264,16 +280,35 @@ inline void ShenandoahMark::mark_through_ref(T *p, ShenandoahObjToScanQueue* q, shenandoah_assert_not_forwarded(p, obj); shenandoah_assert_not_in_cset_except(p, obj, ShenandoahHeap::heap()->cancelled_gc()); - if (in_generation(obj)) { mark_ref(q, mark_context, weak, obj); shenandoah_assert_marked(p, obj); + if (ShenandoahHeap::heap()->mode()->is_generational()) { + // TODO: As implemented herein, GLOBAL collections reconstruct the card table during GLOBAL concurrent + // marking. Note that the card table is cleaned at init_mark time so it needs to be reconstructed to support + // future young-gen collections. It might be better to reconstruct card table in + // ShenandoahHeapRegion::global_oop_iterate_and_fill_dead. We could either mark all live memory as dirty, or could + // use the GLOBAL update-refs scanning of pointers to determine precisely which cards to flag as dirty. + // + if ((GENERATION == YOUNG) && ShenandoahHeap::heap()->is_in(p) && ShenandoahHeap::heap()->is_in_old(p)) { + RememberedScanner* scanner = ShenandoahHeap::heap()->card_scan(); + // Mark card as dirty because remembered set scanning still finds interesting pointer. + ShenandoahHeap::heap()->mark_card_as_dirty((HeapWord*)p); + } else if ((GENERATION == GLOBAL) && in_generation(obj) && + ShenandoahHeap::heap()->is_in(p) && ShenandoahHeap::heap()->is_in_old(p)) { + RememberedScanner* scanner = ShenandoahHeap::heap()->card_scan(); + // Mark card as dirty because GLOBAL marking finds interesting pointer. + ShenandoahHeap::heap()->mark_card_as_dirty((HeapWord*)p); + } + } } else if (old != nullptr) { // Young mark, bootstrapping old or concurrent with old marking. mark_ref(old, mark_context, weak, obj); shenandoah_assert_marked(p, obj); } else if (GENERATION == OLD) { // Old mark, found a young pointer. + // TODO: Rethink this: may be redundant with dirtying of cards identified during young-gen remembered set scanning + // and by mutator write barriers. Assert assert(ShenandoahHeap::heap()->is_in_young(obj), "Expected young object."); ShenandoahHeap::heap()->mark_card_as_dirty(p); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkBitMap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkBitMap.cpp index 75af01c0cc782..c81137298e540 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkBitMap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkBitMap.cpp @@ -43,14 +43,37 @@ size_t ShenandoahMarkBitMap::mark_distance() { return MinObjAlignmentInBytes * BitsPerByte / 2; } +bool ShenandoahMarkBitMap::is_bitmap_clear_range(const HeapWord* start, const HeapWord* end) const { + // Similar to get_next_marked_addr(), without assertion. + // Round addr up to a possible object boundary to be safe. + if (start == end) { + return true; + } + size_t const addr_offset = address_to_index(align_up(start, HeapWordSize << LogMinObjAlignment)); + size_t const limit_offset = address_to_index(end); + size_t const next_offset = get_next_one_offset(addr_offset, limit_offset); + HeapWord* result = index_to_address(next_offset); + return (result == end); +} + + HeapWord* ShenandoahMarkBitMap::get_next_marked_addr(const HeapWord* addr, const HeapWord* limit) const { +#ifdef ASSERT + ShenandoahHeap* heap = ShenandoahHeap::heap(); + ShenandoahHeapRegion* r = heap->heap_region_containing(addr); + ShenandoahMarkingContext* ctx = heap->marking_context(); + HeapWord* tams = ctx->top_at_mark_start(r); assert(limit != NULL, "limit must not be NULL"); + assert(limit <= tams, "limit must be less than TAMS"); +#endif + // Round addr up to a possible object boundary to be safe. size_t const addr_offset = address_to_index(align_up(addr, HeapWordSize << LogMinObjAlignment)); size_t const limit_offset = address_to_index(limit); size_t const nextOffset = get_next_one_offset(addr_offset, limit_offset); - return index_to_address(nextOffset); + HeapWord* result = index_to_address(nextOffset); + return result; } void ShenandoahMarkBitMap::clear_range_within_word(idx_t beg, idx_t end) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkBitMap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkBitMap.hpp index a71cad75baedd..5e9280abcaf6d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkBitMap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkBitMap.hpp @@ -159,6 +159,8 @@ class ShenandoahMarkBitMap { inline bool is_marked_strong(HeapWord* w) const; inline bool is_marked_weak(HeapWord* addr) const; + bool is_bitmap_clear_range(const HeapWord* start, const HeapWord* end) const; + // Return the address corresponding to the next marked bit at or after // "addr", and before "limit", if "limit" is non-NULL. If there is no // such bit, returns "limit" if that is non-NULL, or else "endWord()". diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.cpp index 40dc4549f738f..603ad9e819f5a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.cpp @@ -86,3 +86,10 @@ void ShenandoahFinalMarkUpdateRegionStateClosure::heap_region_do(ShenandoahHeapR } } +ShenandoahCaptureUpdateWaterMarkForOld::ShenandoahCaptureUpdateWaterMarkForOld(ShenandoahMarkingContext* ctx) : + _ctx(ctx), _lock(ShenandoahHeap::heap()->lock()) {} + +void ShenandoahCaptureUpdateWaterMarkForOld::heap_region_do(ShenandoahHeapRegion* r) { + // Remember limit for updating refs. It's guaranteed that we get no from-space-refs written from here on. + r->set_update_watermark_at_safepoint(r->top()); +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.hpp index 31817c61f0d67..6aa013fb3dc0c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.hpp @@ -42,4 +42,16 @@ class ShenandoahFinalMarkUpdateRegionStateClosure : public ShenandoahHeapRegionC bool is_thread_safe() { return true; } }; +class ShenandoahCaptureUpdateWaterMarkForOld : public ShenandoahHeapRegionClosure { +private: + ShenandoahMarkingContext* const _ctx; + ShenandoahHeapLock* const _lock; +public: + ShenandoahCaptureUpdateWaterMarkForOld(ShenandoahMarkingContext* ctx); + + void heap_region_do(ShenandoahHeapRegion* r); + + bool is_thread_safe() { return true; } +}; + #endif // SHARE_GC_SHENANDOAH_SHENANDOAHMARKCLOSURES_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.cpp index 85350e9a3eece..143345c2b236b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.cpp @@ -40,33 +40,62 @@ bool ShenandoahMarkingContext::is_bitmap_clear() const { size_t num_regions = heap->num_regions(); for (size_t idx = 0; idx < num_regions; idx++) { ShenandoahHeapRegion* r = heap->get_region(idx); - if (heap->is_bitmap_slice_committed(r) && !is_bitmap_clear_range(r->bottom(), r->end())) { + if ((r->affiliation() != FREE) && heap->is_bitmap_slice_committed(r) && !is_bitmap_clear_range(r->bottom(), r->end())) { return false; } } return true; } -bool ShenandoahMarkingContext::is_bitmap_clear_range(HeapWord* start, HeapWord* end) const { - return _mark_bit_map.get_next_marked_addr(start, end) == end; +bool ShenandoahMarkingContext::is_bitmap_clear_range(const HeapWord* start, const HeapWord* end) const { + if (start < end) { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + size_t start_idx = heap->heap_region_index_containing(start); + size_t end_idx = heap->heap_region_index_containing(end - 1); + while (start_idx <= end_idx) { + ShenandoahHeapRegion* r = heap->get_region(start_idx); + if (!heap->is_bitmap_slice_committed(r)) + return true; + start_idx++; + } + } + return _mark_bit_map.is_bitmap_clear_range(start, end); } void ShenandoahMarkingContext::initialize_top_at_mark_start(ShenandoahHeapRegion* r) { size_t idx = r->index(); HeapWord *bottom = r->bottom(); + _top_at_mark_starts_base[idx] = bottom; - _top_bitmaps[idx] = bottom; + // Arrange that the first time we use this bitmap, we clean from bottom to end. + _top_bitmaps[idx] = r->end(); + + log_debug(gc)("SMC:initialize_top_at_mark_start for region [" PTR_FORMAT ", " PTR_FORMAT "], top_bitmaps set to " PTR_FORMAT, + p2i(r->bottom()), p2i(r->end()), p2i(r->end())); +} + +HeapWord* ShenandoahMarkingContext::top_bitmap(ShenandoahHeapRegion* r) { + return _top_bitmaps[r->index()]; } void ShenandoahMarkingContext::clear_bitmap(ShenandoahHeapRegion* r) { HeapWord* bottom = r->bottom(); HeapWord* top_bitmap = _top_bitmaps[r->index()]; - if (top_bitmap > bottom) { - _mark_bit_map.clear_range_large(MemRegion(bottom, top_bitmap)); - _top_bitmaps[r->index()] = bottom; + + log_debug(gc)("SMC:clear_bitmap for %s region [" PTR_FORMAT ", " PTR_FORMAT "], top_bitmap: " PTR_FORMAT, + affiliation_name(r->affiliation()), p2i(r->bottom()), p2i(r->end()), p2i(top_bitmap)); + + if (r->affiliation() != FREE) { + if (top_bitmap > bottom) { + _mark_bit_map.clear_range_large(MemRegion(bottom, top_bitmap)); + _top_bitmaps[r->index()] = bottom; + } + r->clear_live_data(); + assert(is_bitmap_clear_range(bottom, r->end()), + "Region " SIZE_FORMAT " should have no marks in bitmap", r->index()); } - assert(is_bitmap_clear_range(bottom, r->end()), - "Region " SIZE_FORMAT " should have no marks in bitmap", r->index()); + // heap iterators include FREE regions, which don't need to be cleared. + // TODO: would be better for certain iterators to not include FREE regions. } bool ShenandoahMarkingContext::is_complete() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.hpp index b39be1fb2d659..05948d6aff92b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.hpp @@ -75,11 +75,13 @@ class ShenandoahMarkingContext : public CHeapObj { inline void reset_top_at_mark_start(ShenandoahHeapRegion* r); void initialize_top_at_mark_start(ShenandoahHeapRegion* r); + HeapWord* top_bitmap(ShenandoahHeapRegion* r); + inline void reset_top_bitmap(ShenandoahHeapRegion *r); void clear_bitmap(ShenandoahHeapRegion *r); bool is_bitmap_clear() const; - bool is_bitmap_clear_range(HeapWord* start, HeapWord* end) const; + bool is_bitmap_clear_range(const HeapWord* start, const HeapWord* end) const; bool is_complete(); void mark_complete(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.inline.hpp index 0bf9e500d0f50..d3367d0233e87 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.inline.hpp @@ -73,19 +73,38 @@ inline bool ShenandoahMarkingContext::allocated_after_mark_start(HeapWord* addr) } inline void ShenandoahMarkingContext::capture_top_at_mark_start(ShenandoahHeapRegion *r) { - size_t idx = r->index(); - HeapWord* old_tams = _top_at_mark_starts_base[idx]; - HeapWord* new_tams = r->top(); - - assert(new_tams >= old_tams, - "Region " SIZE_FORMAT", TAMS updates should be monotonic: " PTR_FORMAT " -> " PTR_FORMAT, - idx, p2i(old_tams), p2i(new_tams)); - assert(is_bitmap_clear_range(old_tams, new_tams), - "Region " SIZE_FORMAT ", bitmap should be clear while adjusting TAMS: " PTR_FORMAT " -> " PTR_FORMAT, - idx, p2i(old_tams), p2i(new_tams)); - - _top_at_mark_starts_base[idx] = new_tams; - _top_bitmaps[idx] = new_tams; + if (r->affiliation() != FREE) { + size_t idx = r->index(); + HeapWord* old_tams = _top_at_mark_starts_base[idx]; + HeapWord* new_tams = r->top(); + + assert(new_tams >= old_tams, + "Region " SIZE_FORMAT", TAMS updates should be monotonic: " PTR_FORMAT " -> " PTR_FORMAT, + idx, p2i(old_tams), p2i(new_tams)); + assert((new_tams == r->bottom()) || (old_tams == r->bottom()) || (new_tams >= _top_bitmaps[idx]), + "Region " SIZE_FORMAT", top_bitmaps updates should be monotonic: " PTR_FORMAT " -> " PTR_FORMAT, + idx, p2i(_top_bitmaps[idx]), p2i(new_tams)); + assert(old_tams == r->bottom() || is_bitmap_clear_range(old_tams, new_tams), + "Region " SIZE_FORMAT ", bitmap should be clear while adjusting TAMS: " PTR_FORMAT " -> " PTR_FORMAT, + idx, p2i(old_tams), p2i(new_tams)); + + log_debug(gc)("Capturing TAMS for %s Region " SIZE_FORMAT ", was: %llx, now: %llx\n", + affiliation_name(r->affiliation()), idx, (unsigned long long) old_tams, (unsigned long long) new_tams); + + if ((old_tams == r->bottom()) && (new_tams > old_tams)) { + log_debug(gc)("Clearing mark bitmap for %s Region " SIZE_FORMAT " while capturing TAMS", + affiliation_name(r->affiliation()), idx); + + clear_bitmap(r); + } + + _top_at_mark_starts_base[idx] = new_tams; + if (new_tams > r->bottom()) { + // In this case, new_tams is greater than old _top_bitmaps[idx] + _top_bitmaps[idx] = new_tams; + } + } + // else, FREE regions do not need their TAMS updated } inline void ShenandoahMarkingContext::reset_top_at_mark_start(ShenandoahHeapRegion* r) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp index 08586e18ce9ee..a533ccb0883f7 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp @@ -36,8 +36,6 @@ class ShenandoahConcurrentCoalesceAndFillTask : public AbstractGangTask { private: - // remember nworkers, coalesce_and_fill_region_array,coalesce_and_fill_regions_count - uint _nworkers; ShenandoahHeapRegion** _coalesce_and_fill_region_array; uint _coalesce_and_fill_region_count; @@ -67,7 +65,7 @@ class ShenandoahConcurrentCoalesceAndFillTask : public AbstractGangTask { ShenandoahOldGC::ShenandoahOldGC(ShenandoahGeneration* generation, ShenandoahSharedFlag& allow_preemption) : - ShenandoahConcurrentGC(generation), _allow_preemption(allow_preemption) { + ShenandoahConcurrentGC(generation, false), _allow_preemption(allow_preemption) { _coalesce_and_fill_region_array = NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, ShenandoahHeap::heap()->num_regions(), mtGC); } @@ -78,6 +76,47 @@ void ShenandoahOldGC::entry_old_evacuations() { old_heuristics->start_old_evacuations(); } + +// Final mark for old-gen is different than for young or old, so we +// override the implementation. +void ShenandoahOldGC::op_final_mark() { + + ShenandoahHeap* const heap = ShenandoahHeap::heap(); + assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Should be at safepoint"); + assert(!heap->has_forwarded_objects(), "No forwarded objects on this path"); + + if (ShenandoahVerify) { + heap->verifier()->verify_roots_no_forwarded(); + } + + if (!heap->cancelled_gc()) { + assert(_mark.generation()->generation_mode() == OLD, "Generation of Old-Gen GC should be OLD"); + _mark.finish_mark(); + assert(!heap->cancelled_gc(), "STW mark cannot OOM"); + + // Believe notifying JVMTI that the tagmap table will need cleaning is not relevant following old-gen mark + // so commenting out for now: + // JvmtiTagMap::set_needs_cleaning(); + + { + ShenandoahGCPhase phase(ShenandoahPhaseTimings::choose_cset); + ShenandoahHeapLocker locker(heap->lock()); + // Old-gen choose_collection_set() does not directly manipulate heap->collection_set() so no need to clear it. + _generation->heuristics()->choose_collection_set(nullptr, nullptr); + } + + // Believe verification following old-gen concurrent mark needs to be different than verification following + // young-gen concurrent mark, so am commenting this out for now: + // if (ShenandoahVerify) { + // heap->verifier()->verify_after_concmark(); + // } + + if (VerifyAfterGC) { + Universe::verify(); + } + } +} + bool ShenandoahOldGC::collect(GCCause::Cause cause) { ShenandoahHeap* heap = ShenandoahHeap::heap(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.hpp index 55323f283311a..f4de5673d665c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.hpp @@ -27,6 +27,7 @@ #include "gc/shared/gcCause.hpp" #include "gc/shenandoah/shenandoahConcurrentGC.hpp" +#include "gc/shenandoah/shenandoahVerifier.hpp" class ShenandoahGeneration; @@ -34,6 +35,10 @@ class ShenandoahOldGC : public ShenandoahConcurrentGC { public: ShenandoahOldGC(ShenandoahGeneration* generation, ShenandoahSharedFlag& allow_preemption); bool collect(GCCause::Cause cause); + + protected: + virtual void op_final_mark(); + private: ShenandoahHeapRegion** _coalesce_and_fill_region_array; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp index 63dfc28caccfc..c24663922d844 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp @@ -230,4 +230,23 @@ class ShenandoahConcUpdateRefsClosure : public ShenandoahUpdateRefsSuperClosure virtual void do_oop(oop* p) { work(p); } }; +class ShenandoahVerifyRemSetClosure : public BasicOopIterateClosure { + protected: + bool _init_mark; + ShenandoahHeap* _heap; + RememberedScanner* _scanner; + + public: +// Argument distinguishes between initial mark or start of update refs verification. + ShenandoahVerifyRemSetClosure(bool init_mark) : + _init_mark(init_mark), + _heap(ShenandoahHeap::heap()), + _scanner(_heap->card_scan()) { } + template + inline void work(T* p); + + virtual void do_oop(narrowOop* p) { work(p); } + virtual void do_oop(oop* p) { work(p); } +}; + #endif // SHARE_GC_SHENANDOAH_SHENANDOAHOOPCLOSURES_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp index 560cb0932f596..916b6c230ce73 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp @@ -52,4 +52,22 @@ inline void ShenandoahConcUpdateRefsClosure::work(T* p) { _heap->conc_update_with_forwarded(p); } +template +inline void ShenandoahVerifyRemSetClosure::work(T* p) { + T o = RawAccess<>::oop_load(p); + if (!CompressedOops::is_null(o)) { + oop obj = CompressedOops::decode_not_null(o); + if (_heap->is_in_young(obj)) { + size_t card_index = _scanner->card_index_for_addr((HeapWord*) p); + if (_init_mark && !_scanner->is_card_dirty(card_index)) { + ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, NULL, + "Verify init-mark remembered set violation", "clean card should be dirty", __FILE__, __LINE__); + } else if (!_init_mark && !_scanner->is_write_card_dirty(card_index)) { + ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, NULL, + "Verify init-update-refs remembered set violation", "clean card should be dirty", __FILE__, __LINE__); + } + } + } +} + #endif // SHARE_GC_SHENANDOAH_SHENANDOAHOOPCLOSURES_INLINE_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp b/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp index 787774824eccb..99c410dad2ca7 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp @@ -386,12 +386,22 @@ template oop ShenandoahReferenceProcessor::drop(oop reference, ReferenceType type) { log_trace(gc, ref)("Dropped Reference: " PTR_FORMAT " (%s)", p2i(reference), reference_type_name(type)); + ShenandoahHeap* heap = ShenandoahHeap::heap(); + oop referent = reference_referent(reference); assert(reference_referent(reference) == NULL || - ShenandoahHeap::heap()->marking_context()->is_marked(reference_referent(reference)), "only drop references with alive referents"); + heap->marking_context()->is_marked(reference_referent(reference)), "only drop references with alive referents"); // Unlink and return next in list oop next = reference_discovered(reference); reference_set_discovered(reference, NULL); + // When this reference was discovered, it would not have been marked. If it ends up surviving + // the cycle, we need to dirty the card if the reference is old and the referent is young. Note + // that if the reference is not dropped, then its pointer to the referent will be nulled before + // evacuation begins so card does not need to be dirtied. + if (heap->mode()->is_generational() && heap->is_old(reference) && heap->is_in_young(referent)) { + // Note: would be sufficient to mark only the card that holds the start of this Reference object. + heap->card_scan()->mark_range_as_dirty(cast_from_oop(reference), reference->size()); + } return next; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp index b2bfa191619f2..38cdddba7c045 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp @@ -31,7 +31,7 @@ #include "gc/shenandoah/shenandoahReferenceProcessor.hpp" #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp" -ShenandoahDirectCardMarkRememberedSet::ShenandoahDirectCardMarkRememberedSet(CardTable* card_table, size_t total_card_count) { +ShenandoahDirectCardMarkRememberedSet::ShenandoahDirectCardMarkRememberedSet(ShenandoahCardTable* card_table, size_t total_card_count) { _heap = ShenandoahHeap::heap(); _card_table = card_table; _total_card_count = total_card_count; @@ -104,7 +104,6 @@ void ShenandoahScanRememberedTask::work(uint worker_id) { // set up thread local closure for shen ref processor _rp->set_mark_closure(worker_id, &cl); - ShenandoahHeapRegion* region = _regions->next(); while (region != NULL) { if (region->affiliation() == OLD_GENERATION) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp index 3cab558bacf89..a296f842fefe4 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp @@ -236,7 +236,7 @@ class ShenandoahDirectCardMarkRememberedSet: public CHeapObj { // CardTable::dirty_card_val() ShenandoahHeap *_heap; - CardTable *_card_table; + ShenandoahCardTable *_card_table; size_t _card_shift; size_t _total_card_count; size_t _cluster_count; @@ -247,9 +247,11 @@ class ShenandoahDirectCardMarkRememberedSet: public CHeapObj { uint8_t *_overreach_map; // Points to first entry within the overreach card table uint8_t *_overreach_map_base; // Points to overreach_map minus the bias computed from address of heap memory + uint64_t _wide_clean_value; + public: // count is the number of cards represented by the card table. - ShenandoahDirectCardMarkRememberedSet(CardTable *card_table, size_t total_card_count); + ShenandoahDirectCardMarkRememberedSet(ShenandoahCardTable *card_table, size_t total_card_count); ~ShenandoahDirectCardMarkRememberedSet(); // Card index is zero-based relative to _byte_map. @@ -257,9 +259,11 @@ class ShenandoahDirectCardMarkRememberedSet: public CHeapObj { size_t card_index_for_addr(HeapWord *p); HeapWord *addr_for_card_index(size_t card_index); bool is_card_dirty(size_t card_index); + bool is_write_card_dirty(size_t card_index); void mark_card_as_dirty(size_t card_index); void mark_range_as_dirty(size_t card_index, size_t num_cards); void mark_card_as_clean(size_t card_index); + void mark_read_card_as_clean(size_t card_index); void mark_range_as_clean(size_t card_index, size_t num_cards); void mark_overreach_card_as_dirty(size_t card_index); bool is_card_dirty(HeapWord *p); @@ -277,6 +281,32 @@ class ShenandoahDirectCardMarkRememberedSet: public CHeapObj { // Called by GC thread at end of concurrent mark or evacuation phase. Each parallel GC thread typically merges different // subranges of all overreach entries. void merge_overreach(size_t first_cluster, size_t count); + + // Called by GC thread at start of concurrent mark to exchange roles of read and write remembered sets. + // Not currently used because mutator write barrier does not honor changes to the location of card table. + void swap_remset() { _card_table->swap_card_tables(); } + + HeapWord* whole_heap_base() { return _whole_heap_base; } + HeapWord* whole_heap_end() { return _whole_heap_end; } + + // Instead of swap_remset, the current implementation of concurrent remembered set scanning does reset_remset + // in parallel threads, each invocation processing one entire HeapRegion at a time. Processing of a region + // consists of copying the write table to the read table and cleaning the write table. + void reset_remset(HeapWord* start, size_t word_count) { + size_t card_index = card_index_for_addr(start); + size_t num_cards = word_count / CardTable::card_size_in_words; + size_t iterations = num_cards / (sizeof (intptr_t) / sizeof (CardTable::CardValue)); + intptr_t* read_table_ptr = (intptr_t*) &(_card_table->read_byte_map())[card_index]; + intptr_t* write_table_ptr = (intptr_t*) &(_card_table->write_byte_map())[card_index]; + for (size_t i = 0; i < iterations; i++) { + *read_table_ptr++ = *write_table_ptr; + *write_table_ptr++ = CardTable::clean_card_row_val(); + } + } + + // Called by GC thread after scanning old remembered set in order to prepare for next GC pass + void clear_old_remset() { _card_table->clear_read_table(); } + }; // A ShenandoahCardCluster represents the minimal unit of work @@ -647,7 +677,7 @@ class ShenandoahCardCluster: public CHeapObj { // It is not necessary to invoke register_object at the very instant // an object is allocated. It is only necessary to invoke it // prior to the next start of a garbage collection concurrent mark - // or concurrent evacuation phase. An "ideal" time to register + // or concurrent update-references phase. An "ideal" time to register // objects is during post-processing of a GCLAB after the GCLAB is // retired due to depletion of its memory. // @@ -722,26 +752,49 @@ class ShenandoahCardCluster: public CHeapObj { // object_starts information is coherent. - // Synchronization thoughts from kelvin: + // Notes on synchronization of register_object(): + // + // 1. For efficiency, there is no locking in the implementation of register_object() + // 2. Thus, it is required that users of this service assure that concurrent/parallel invocations of + // register_object() do pertain to the same card's memory range. See discussion below to undestand + // the risks. + // 3. When allocating from a TLAB or GCLAB, the mutual exclusion can be guaranteed by assuring that each + // LAB's start and end are aligned on card memory boundaries. + // 4. Use the same lock that guarantees exclusivity when performing free-list allocation within heap regions. // - // previously, I had contemplated a more complex implementation of - // object registration, which had to touch every card spanned by the - // registered object. But the current implementation is much simpler, - // and only has to touch the card that contains the start of the - // object. + // Register the newly allocated object while we're holding the global lock since there's no synchronization + // built in to the implementation of register_object(). There are potential races when multiple independent + // threads are allocating objects, some of which might span the same card region. For example, consider + // a card table's memory region within which three objects are being allocated by three different threads: + // + // objects being "concurrently" allocated: + // [-----a------][-----b-----][--------------c------------------] + // [---- card table memory range --------------] + // + // Before any objects are allocated, this card's memory range holds no objects. Note that: + // allocation of object a wants to set the has-object, first-start, and last-start attributes of the preceding card region. + // allocation of object b wants to set the has-object, first-start, and last-start attributes of this card region. + // allocation of object c also wants to set the has-object, first-start, and last-start attributes of this card region. + // + // The thread allocating b and the thread allocating c can "race" in various ways, resulting in confusion, such as last-start + // representing object b while first-start represents object c. This is why we need to require all register_object() + // invocations associated with objects that are allocated from "free lists" to provide their own mutual exclusion locking + // mechanism. - // if I were careful to assure that every GCLAB aligns with the start - // of a card region and spanned a multiple of the card region size, - // then the post-processing of each GCLAB with regards to - // register_object() invocations can proceed without synchronization. + // Reset the has_object() information to false for all cards in the range between from and to. + void reset_object_range(HeapWord *from, HeapWord *to); - // But right now, we're not even using GCLABs We are doing shared - // allocations. But, we must hold a lock while we are doing these, so - // maybe I just piggy back on the lock that we already hold for - // managing the free lists and register each object newly allocated by - // the shared allocator. + // register_object() requires that the caller hold the heap lock + // before calling it. void register_object(HeapWord* address); + // register_object_wo_lock() does not require that the caller hold + // the heap lock before calling it, under the assumption that the + // caller has assure no other thread will endeavor to concurrently + // register objects that start within the same card's memory region + // as address. + void register_object_wo_lock(HeapWord* address); + // During the reference updates phase of GC, we walk through each old-gen memory region that was // not part of the collection set and we invalidate all unmarked objects. As part of this effort, // we coalesce neighboring dead objects in order to make future remembered set scanning more @@ -752,24 +805,24 @@ class ShenandoahCardCluster: public CHeapObj { // At some future time, we may implement a further optimization: satisfy future allocation requests // by carving new objects out of the range of memory that represents the coalesced dead objects. // - // In its current implementation, unregister_object() serves the needs of coalescing objects. - // - // Suppose we want to combine several dead objects into a single coalesced object. How does this // impact our representation of crossing map information? - // 1. If the newly coalesced region is contained entirely within a single region, that region's last + // 1. If the newly coalesced range is contained entirely within a card range, that card's last // start entry either remains the same or it is changed to the start of the coalesced region. - // 2. For the region that holds the start of the coalesced object, it will not impact the first start + // 2. For the card that holds the start of the coalesced object, it will not impact the first start // but it may impact the last start. - // 3. For following regions spanned entirely by the newly coalesced object, it will change has_object + // 3. For following cards spanned entirely by the newly coalesced object, it will change has_object // to false (and make first-start and last-start "undefined"). - // 4. For a following region that is spanned patially by the newly coalesced object, it may change + // 4. For a following card that is spanned patially by the newly coalesced object, it may change // first-start value, but it will not change the last-start value. // // The range of addresses represented by the arguments to coalesce_objects() must represent a range // of memory that was previously occupied exactly by one or more previously registered objects. For // convenience, it is legal to invoke coalesce_objects() with arguments that span a single previously // registered object. + // + // The role of coalesce_objects is to change the crossing map information associated with all of the coalesced + // objects. void coalesce_objects(HeapWord* address, size_t length_in_words); // The typical use case is going to look something like this: @@ -802,14 +855,6 @@ class ShenandoahCardCluster: public CHeapObj { // ability to scan the old-gen remembered set for references to // objects residing in young-gen memory. // -// In an initial implementation, remembered set scanning happens -// during a HotSpot safepoint. This greatly simplifies the -// implementation and improves efficiency of remembered set scanning, -// but this design choice increases pause times experienced at the -// start of concurrent marking and concurrent evacuation. Pause times -// will be especially long if old-gen memory holds many pointers to -// young-gen memory. -// // Scanning normally begins with an invocation of numRegions and ends // after all clusters of all regions have been scanned. // @@ -874,9 +919,11 @@ class ShenandoahScanRemembered: public CHeapObj { size_t card_index_for_addr(HeapWord *p); HeapWord *addr_for_card_index(size_t card_index); bool is_card_dirty(size_t card_index); + bool is_write_card_dirty(size_t card_index) { return _rs->is_write_card_dirty(card_index); } void mark_card_as_dirty(size_t card_index); void mark_range_as_dirty(size_t card_index, size_t num_cards); void mark_card_as_clean(size_t card_index); + void mark_read_card_as_clean(size_t card_index) { _rs->mark_read_card_clean(card_index); } void mark_range_as_clean(size_t card_index, size_t num_cards); void mark_overreach_card_as_dirty(size_t card_index); bool is_card_dirty(HeapWord *p); @@ -889,10 +936,24 @@ class ShenandoahScanRemembered: public CHeapObj { void initialize_overreach(size_t first_cluster, size_t count); void merge_overreach(size_t first_cluster, size_t count); + // Called by GC thread at start of concurrent mark to exchange roles of read and write remembered sets. + void swap_remset() { _rs->swap_remset(); } + + void reset_remset(HeapWord* start, size_t word_count) { _rs->reset_remset(start, word_count); } + + // Called by GC thread after scanning old remembered set in order to prepare for next GC pass + void clear_old_remset() { _rs->clear_old_remset(); } + size_t cluster_for_addr(HeapWord *addr); + + void reset_object_range(HeapWord *from, HeapWord *to); void register_object(HeapWord *addr); + void register_object_wo_lock(HeapWord *addr); void coalesce_objects(HeapWord *addr, size_t length_in_words); + // Return true iff this object is "properly" registered. + bool verify_registration(HeapWord* address, size_t size_in_words); + // clear the cards to clean, and clear the object_starts info to no objects void mark_range_as_empty(HeapWord *addr, size_t length_in_words); @@ -930,9 +991,15 @@ class ShenandoahScanRemembered: public CHeapObj { template inline void process_clusters(size_t first_cluster, size_t count, HeapWord *end_of_range, ClosureType *oops); + template + inline void process_clusters(size_t first_cluster, size_t count, HeapWord *end_of_range, ClosureType *oops, bool use_write_table); + template inline void process_region(ShenandoahHeapRegion* region, ClosureType *cl); + template + inline void process_region(ShenandoahHeapRegion* region, ClosureType *cl, bool use_write_table); + // To Do: // Create subclasses of ShenandoahInitMarkRootsClosure and // ShenandoahEvacuateUpdateRootsClosure and any other closures diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp index 5ef784cd9596b..93bda125fc95d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp @@ -29,6 +29,7 @@ #include "memory/iterator.hpp" #include "oops/oop.hpp" #include "oops/objArrayOop.hpp" +#include "gc/shared/collectorCounters.hpp" #include "gc/shenandoah/shenandoahCardTable.hpp" #include "gc/shenandoah/shenandoahHeap.hpp" #include "gc/shenandoah/shenandoahHeapRegion.hpp" @@ -49,36 +50,44 @@ ShenandoahDirectCardMarkRememberedSet::addr_for_card_index(size_t card_index) { return _whole_heap_base + CardTable::card_size_in_words * card_index; } +inline bool +ShenandoahDirectCardMarkRememberedSet::is_write_card_dirty(size_t card_index) { + uint8_t *bp = &(_card_table->write_byte_map())[card_index]; + return (bp[0] == CardTable::dirty_card_val()); +} + inline bool ShenandoahDirectCardMarkRememberedSet::is_card_dirty(size_t card_index) { - uint8_t *bp = &_byte_map[card_index]; + uint8_t *bp = &(_card_table->read_byte_map())[card_index]; return (bp[0] == CardTable::dirty_card_val()); } inline void ShenandoahDirectCardMarkRememberedSet::mark_card_as_dirty(size_t card_index) { - uint8_t *bp = &_byte_map[card_index]; + uint8_t *bp = &(_card_table->write_byte_map())[card_index]; bp[0] = CardTable::dirty_card_val(); } inline void ShenandoahDirectCardMarkRememberedSet::mark_range_as_dirty(size_t card_index, size_t num_cards) { - uint8_t *bp = &_byte_map[card_index]; - while (num_cards-- > 0) + uint8_t *bp = &(_card_table->write_byte_map())[card_index]; + while (num_cards-- > 0) { *bp++ = CardTable::dirty_card_val(); + } } inline void ShenandoahDirectCardMarkRememberedSet::mark_card_as_clean(size_t card_index) { - uint8_t *bp = &_byte_map[card_index]; + uint8_t *bp = &(_card_table->write_byte_map())[card_index]; bp[0] = CardTable::clean_card_val(); } inline void ShenandoahDirectCardMarkRememberedSet::mark_range_as_clean(size_t card_index, size_t num_cards) { - uint8_t *bp = &_byte_map[card_index]; - while (num_cards-- > 0) + uint8_t *bp = &(_card_table->write_byte_map())[card_index]; + while (num_cards-- > 0) { *bp++ = CardTable::clean_card_val(); + } } inline void @@ -89,36 +98,55 @@ ShenandoahDirectCardMarkRememberedSet::mark_overreach_card_as_dirty(size_t card_ inline bool ShenandoahDirectCardMarkRememberedSet::is_card_dirty(HeapWord *p) { - uint8_t *bp = &_byte_map_base[uintptr_t(p) >> _card_shift]; + size_t index = card_index_for_addr(p); + uint8_t *bp = &(_card_table->read_byte_map())[index]; return (bp[0] == CardTable::dirty_card_val()); } inline void ShenandoahDirectCardMarkRememberedSet::mark_card_as_dirty(HeapWord *p) { - uint8_t *bp = &_byte_map_base[uintptr_t(p) >> _card_shift]; + size_t index = card_index_for_addr(p); + uint8_t *bp = &(_card_table->write_byte_map())[index]; bp[0] = CardTable::dirty_card_val(); } inline void ShenandoahDirectCardMarkRememberedSet::mark_range_as_dirty(HeapWord *p, size_t num_heap_words) { - uint8_t *bp = &_byte_map_base[uintptr_t(p) >> _card_shift]; - uint8_t *end_bp = &_byte_map_base[uintptr_t(p + num_heap_words) >> _card_shift]; - while (bp < end_bp) + uint8_t *bp = &(_card_table->write_byte_map_base())[uintptr_t(p) >> _card_shift]; + uint8_t *end_bp = &(_card_table->write_byte_map_base())[uintptr_t(p + num_heap_words) >> _card_shift]; + // If (p + num_heap_words) is not aligned on card boundary, we also need to dirty last card. + if (((unsigned long long) (p + num_heap_words)) & (CardTable::card_size - 1)) { + end_bp++; + } + while (bp < end_bp) { *bp++ = CardTable::dirty_card_val(); + } } inline void ShenandoahDirectCardMarkRememberedSet::mark_card_as_clean(HeapWord *p) { - uint8_t *bp = &_byte_map_base[uintptr_t(p) >> _card_shift]; + size_t index = card_index_for_addr(p); + uint8_t *bp = &(_card_table->write_byte_map())[index]; + bp[0] = CardTable::clean_card_val(); +} + +inline void +ShenandoahDirectCardMarkRememberedSet::mark_read_card_as_clean(size_t index) { + uint8_t *bp = &(_card_table->read_byte_map())[index]; bp[0] = CardTable::clean_card_val(); } inline void ShenandoahDirectCardMarkRememberedSet::mark_range_as_clean(HeapWord *p, size_t num_heap_words) { - uint8_t *bp = &_byte_map_base[uintptr_t(p) >> _card_shift]; - uint8_t *end_bp = &_byte_map_base[uintptr_t(p + num_heap_words) >> _card_shift]; - while (bp < end_bp) + uint8_t *bp = &(_card_table->write_byte_map_base())[uintptr_t(p) >> _card_shift]; + uint8_t *end_bp = &(_card_table->write_byte_map_base())[uintptr_t(p + num_heap_words) >> _card_shift]; + // If (p + num_heap_words) is not aligned on card boundary, we also need to clean last card. + if (((unsigned long long) (p + num_heap_words)) & (CardTable::card_size - 1)) { + end_bp++; + } + while (bp < end_bp) { *bp++ = CardTable::clean_card_val(); + } } inline void @@ -132,9 +160,33 @@ ShenandoahDirectCardMarkRememberedSet::cluster_count() { return _cluster_count; } +// No lock required because arguments align with card boundaries. +template +inline void +ShenandoahCardCluster::reset_object_range(HeapWord* from, HeapWord* to) { + assert(((((unsigned long long) from) & (CardTable::card_size - 1)) == 0) && + ((((unsigned long long) to) & (CardTable::card_size - 1)) == 0), + "reset_object_range bounds must align with card boundaries"); + size_t card_at_start = _rs->card_index_for_addr(from); + size_t num_cards = (to - from) / CardTable::card_size_in_words; + + for (size_t i = 0; i < num_cards; i++) { + object_starts[card_at_start + i] = 0; + } +} + +// Assume only one thread at a time registers objects pertaining to +// each card-table entry's range of memory. template inline void ShenandoahCardCluster::register_object(HeapWord* address) { + shenandoah_assert_heaplocked(); + register_object_wo_lock(address); +} + +template +inline void +ShenandoahCardCluster::register_object_wo_lock(HeapWord* address) { size_t card_at_start = _rs->card_index_for_addr(address); HeapWord *card_start_address = _rs->addr_for_card_index(card_at_start); uint8_t offset_in_card = address - card_start_address; @@ -397,6 +449,11 @@ template inline void ShenandoahScanRemembered::merge_overreach(size_t first_cluster, size_t count) { _rs->merge_overreach(first_cluster, count); } +template +inline void +ShenandoahScanRemembered::reset_object_range(HeapWord *from, HeapWord *to) { + _scc->reset_object_range(from, to); +} template inline void @@ -404,6 +461,92 @@ ShenandoahScanRemembered::register_object(HeapWord *addr) { _scc->register_object(addr); } +template +inline void +ShenandoahScanRemembered::register_object_wo_lock(HeapWord *addr) { + _scc->register_object_wo_lock(addr); +} + +template +inline bool +ShenandoahScanRemembered::verify_registration(HeapWord* address, size_t size_in_words) { + + size_t index = card_index_for_addr(address); + if (!_scc->has_object(index)) { + return false; + } + HeapWord* base_addr = addr_for_card_index(index); + size_t offset = _scc->get_first_start(index); + ShenandoahHeap* heap = ShenandoahHeap::heap(); + ShenandoahMarkingContext* ctx; + + if (heap->doing_mixed_evacuations()) { + ctx = heap->marking_context(); + } else { + ctx = nullptr; + } + + // Verify that I can find this object within its enclosing card by scanning forward from first_start. + while (base_addr + offset < address) { + oop obj = oop(base_addr + offset); + if (!ctx || ctx->is_marked(obj)) { + offset += obj->size(); + } else { + // This object is not live so don't trust its size() + ShenandoahHeapRegion* r = heap->heap_region_containing(base_addr + offset); + HeapWord* tams = ctx->top_at_mark_start(r); + if (base_addr + offset >= tams) { + offset += obj->size(); + } else { + offset = ctx->get_next_marked_addr(base_addr + offset, tams) - base_addr; + } + } + } + if (base_addr + offset != address){ + return false; + } + + if (!ctx) { + // Make sure that last_offset is properly set for the enclosing card, but we can't verify this for + // candidate collection-set regions during mixed evacuations, so disable this check in general + // during mixed evacuations. + // + // TODO: could do some additional checking during mixed evacuations if we wanted to work harder. + size_t prev_offset = offset; + do { + HeapWord* obj_addr = base_addr + offset; + oop obj = oop(base_addr + offset); + prev_offset = offset; + offset += obj->size(); + } while (offset < CardTable::card_size_in_words); + if (_scc->get_last_start(index) != prev_offset) { + return false; + } + + // base + offset represents address of first object that starts on following card, if there is one. + + // Notes: base_addr is addr_for_card_index(index) + // base_addr + offset is end of the object we are verifying + // cannot use card_index_for_addr(base_addr + offset) because it asserts arg < end of whole heap + size_t end_card_index = index + offset / CardTable::card_size_in_words; + + // If there is a following object registered, it should begin where this object ends. + if ((base_addr + offset < _rs->whole_heap_end()) && _scc->has_object(end_card_index) && + ((addr_for_card_index(end_card_index) + _scc->get_first_start(end_card_index)) != (base_addr + offset))) { + return false; + } + + // Assure that no other objects are registered "inside" of this one. + for (index++; index < end_card_index; index++) { + if (_scc->has_object(index)) { + return false; + } + } + } + + return true; +} + template inline void ShenandoahScanRemembered::coalesce_objects(HeapWord *addr, size_t length_in_words) { @@ -422,23 +565,48 @@ template inline void ShenandoahScanRemembered::process_clusters(size_t first_cluster, size_t count, HeapWord *end_of_range, ClosureType *cl) { + process_clusters(first_cluster, count, end_of_range, cl, false); +} + +template +template +inline void +ShenandoahScanRemembered::process_clusters(size_t first_cluster, size_t count, HeapWord *end_of_range, + ClosureType *cl, bool write_table) { // Unlike traditional Shenandoah marking, the old-gen resident objects that are examined as part of the remembered set are not // themselves marked. Each such object will be scanned only once. Any young-gen objects referenced from the remembered set will // be marked and then subsequently scanned. + // If old-gen evacuation is active, then MarkingContext for old-gen heap regions is valid. We use the MarkingContext + // bits to determine which objects within a DIRTY card need to be scanned. This is necessary because old-gen heap + // regions which are in the candidate collection set have not been coalesced and filled. Thus, these heap regions + // may contain zombie objects. Zombie objects are known to be dead, but have not yet been "collected". Scanning + // zombie objects is unsafe because the Klass pointer is not reliable, objects referenced from a zombie may have been + // collected and their memory repurposed, and because zombie objects might refer to objects that are themselves dead. + + ShenandoahHeap* heap = ShenandoahHeap::heap(); + ShenandoahOldHeuristics* old_heuristics = heap->old_heuristics(); + ShenandoahMarkingContext* ctx; + + if (heap->doing_mixed_evacuations()) { + ctx = heap->marking_context(); + } else { + ctx = nullptr; + } + + HeapWord* end_of_clusters = _rs->addr_for_card_index(first_cluster) + + count * ShenandoahCardCluster::CardsPerCluster * CardTable::card_size_in_words; while (count-- > 0) { size_t card_index = first_cluster * ShenandoahCardCluster::CardsPerCluster; size_t end_card_index = card_index + ShenandoahCardCluster::CardsPerCluster; - first_cluster++; size_t next_card_index = 0; while (card_index < end_card_index) { - - bool is_dirty = _rs->is_card_dirty(card_index); + bool is_dirty = (write_table)? is_write_card_dirty(card_index): is_card_dirty(card_index); bool has_object = _scc->has_object(card_index); - if (is_dirty) { + size_t prev_card_index = card_index; if (has_object) { // Scan all objects that start within this card region. size_t start_offset = _scc->get_first_start(card_index); @@ -461,34 +629,48 @@ ShenandoahScanRemembered::process_clusters(size_t first_cluster, while (p < endp) { oop obj = oop(p); - // Future TODO: - // For improved efficiency, we might want to give special handling of obj->is_objArray(). In - // particular, in that case, we might want to divide the effort for scanning of a very long object array - // between multiple threads. - if (obj->is_objArray()) { - objArrayOop array = objArrayOop(obj); - int len = array->length(); - array->oop_iterate_range(cl, 0, len); - } else if (obj->is_instance()) { - obj->oop_iterate(cl); + // ctx->is_marked() returns true if mark bit set or if obj above TAMS. + if (!ctx || ctx->is_marked(obj)) { + // Future TODO: + // For improved efficiency, we might want to give special handling of obj->is_objArray(). In + // particular, in that case, we might want to divide the effort for scanning of a very long object array + // between multiple threads. + if (obj->is_objArray()) { + objArrayOop array = objArrayOop(obj); + int len = array->length(); + array->oop_iterate_range(cl, 0, len); + } else if (obj->is_instance()) { + obj->oop_iterate(cl); + } else { + // Case 3: Primitive array. Do nothing, no oops there. We use the same + // performance tweak TypeArrayKlass::oop_oop_iterate_impl is using: + // We skip iterating over the klass pointer since we know that + // Universe::TypeArrayKlass never moves. + assert (obj->is_typeArray(), "should be type array"); + } + p += obj->size(); } else { - // Case 3: Primitive array. Do nothing, no oops there. We use the same - // performance tweak TypeArrayKlass::oop_oop_iterate_impl is using: - // We skip iterating over the klass pointer since we know that - // Universe::TypeArrayKlass never moves. - assert (obj->is_typeArray(), "should be type array"); + // This object is not marked so we don't scan it. + ShenandoahHeapRegion* r = heap->heap_region_containing(p); + HeapWord* tams = ctx->top_at_mark_start(r); + if (p >= tams) { + p += obj->size(); + } else { + p = ctx->get_next_marked_addr(p, tams); + } } - p += obj->size(); } - if (p > endp) + if (p > endp) { card_index = card_index + (p - card_start) / CardTable::card_size_in_words; - else // p == endp + } else { // p == endp card_index = next_card_index; + } } else { - // otherwise, this card will have been scanned during scan of a previous cluster. + // Card is dirty but has no object. Card will have been scanned during scan of a previous cluster. card_index++; } } else if (has_object) { + // Card is clean but has object. // Scan the last object that starts within this card memory if it spans at least one dirty card within this cluster // or if it reaches into the next cluster. @@ -496,42 +678,59 @@ ShenandoahScanRemembered::process_clusters(size_t first_cluster, HeapWord *card_start = _rs->addr_for_card_index(card_index); HeapWord *p = card_start + start_offset; oop obj = oop(p); - HeapWord *nextp = p + obj->size(); - // Can't use _scc->card_index_for_addr(endp) here because it crashes with assertion - // failure if nextp points to end of heap. - size_t last_card = card_index + (nextp - card_start) / CardTable::card_size_in_words; + size_t last_card; + if (!ctx || ctx->is_marked(obj)) { + HeapWord *nextp = p + obj->size(); - bool reaches_next_cluster = (last_card > end_card_index); - bool spans_dirty_within_this_cluster = false; + // Can't use _scc->card_index_for_addr(endp) here because it crashes with assertion + // failure if nextp points to end of heap. + last_card = card_index + (nextp - card_start) / CardTable::card_size_in_words; - if (!reaches_next_cluster) { - size_t span_card; - for (span_card = card_index+1; span_card <= last_card; span_card++) - if (_rs->is_card_dirty(span_card)) { - spans_dirty_within_this_cluster = true; - break; - } - } + bool reaches_next_cluster = (last_card > end_card_index); + bool spans_dirty_within_this_cluster = false; - if (reaches_next_cluster || spans_dirty_within_this_cluster) { - if (obj->is_objArray()) { - objArrayOop array = objArrayOop(obj); - int len = array->length(); - array->oop_iterate_range(cl, 0, len); - } else if (obj->is_instance()) { - obj->oop_iterate(cl); + if (!reaches_next_cluster) { + size_t span_card; + for (span_card = card_index+1; span_card <= last_card; span_card++) + if ((write_table)? _rs->is_write_card_dirty(span_card): _rs->is_card_dirty(span_card)) { + spans_dirty_within_this_cluster = true; + break; + } + } + + if (reaches_next_cluster || spans_dirty_within_this_cluster) { + if (obj->is_objArray()) { + objArrayOop array = objArrayOop(obj); + int len = array->length(); + array->oop_iterate_range(cl, 0, len); + } else if (obj->is_instance()) { + obj->oop_iterate(cl); + } else { + // Case 3: Primitive array. Do nothing, no oops there. We use the same + // performance tweak TypeArrayKlass::oop_oop_iterate_impl is using: + // We skip iterating over the klass pointer since we know that + // Universe::TypeArrayKlass never moves. + assert (obj->is_typeArray(), "should be type array"); + } + } + } else { + // The object that spans end of this clean card is not marked, so no need to scan it or its + // unmarked neighbors. + ShenandoahHeapRegion* r = heap->heap_region_containing(p); + HeapWord* tams = ctx->top_at_mark_start(r); + HeapWord* nextp; + if (p >= tams) { + nextp = p + obj->size(); } else { - // Case 3: Primitive array. Do nothing, no oops there. We use the same - // performance tweak TypeArrayKlass::oop_oop_iterate_impl is using: - // We skip iterating over the klass pointer since we know that - // Universe::TypeArrayKlass never moves. - assert (obj->is_typeArray(), "should be type array"); + nextp = ctx->get_next_marked_addr(p, tams); } + last_card = card_index + (nextp - card_start) / CardTable::card_size_in_words; } // Increment card_index to account for the spanning object, even if we didn't scan it. card_index = (last_card > card_index)? last_card: card_index + 1; } else { + // Card is clean and has no object. No need to clean this card. card_index++; } } @@ -542,6 +741,13 @@ template template inline void ShenandoahScanRemembered::process_region(ShenandoahHeapRegion *region, ClosureType *cl) { + process_region(region, cl, false); +} + +template +template +inline void +ShenandoahScanRemembered::process_region(ShenandoahHeapRegion *region, ClosureType *cl, bool use_write_table) { HeapWord *start_of_range = region->bottom(); size_t start_cluster_no = cluster_for_addr(start_of_range); @@ -550,19 +756,29 @@ ShenandoahScanRemembered::process_region(ShenandoahHeapRegion *re // // region->top() represents the end of allocated memory within this region. Any addresses // beyond region->top() should not be scanned as that memory does not hold valid objects. - HeapWord *end_of_range = region->top(); - // end_of_range may point to the middle of a cluster because region->top() may be different than region->end. + HeapWord *end_of_range; + if (use_write_table) { + // This is update-refs servicing. + end_of_range = region->get_update_watermark(); + } else { + // This is concurrent mark servicing. Note that TAMS for this region is TAMS at start of old-gen + // collection. Here, we need to scan up to TAMS for most recently initiated young-gen collection. + // Since all LABs are retired at init mark, and since replacement LABs are allocated lazily, and since no + // promotions occur until evacuation phase, TAMS for most recent young-gen is same as top(). + end_of_range = region->top(); + } + + // end_of_range may point to the middle of a cluster because region->top() may be different than region->end(). // We want to assure that our process_clusters() request spans all relevant clusters. Note that each cluster // processed will avoid processing beyond end_of_range. size_t num_heapwords = end_of_range - start_of_range; - unsigned int cluster_size = CardTable::card_size_in_words * - ShenandoahCardCluster::CardsPerCluster; + unsigned int cluster_size = CardTable::card_size_in_words * ShenandoahCardCluster::CardsPerCluster; size_t num_clusters = (size_t) ((num_heapwords - 1 + cluster_size) / cluster_size); // Remembered set scanner - process_clusters(start_cluster_no, num_clusters, end_of_range, cl); + process_clusters(start_cluster_no, num_clusters, end_of_range, cl, use_write_table); } template diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp index 3f989df6f3b0b..ae00499a33308 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp @@ -59,10 +59,12 @@ class VM_ShenandoahReferenceOperation : public VM_ShenandoahOperation { class VM_ShenandoahInitMark: public VM_ShenandoahOperation { private: ShenandoahConcurrentGC* const _gc; + const bool _do_old_gc_bootstrap; public: - VM_ShenandoahInitMark(ShenandoahConcurrentGC* gc) : + VM_ShenandoahInitMark(ShenandoahConcurrentGC* gc, bool do_old_gc_bootstrap) : VM_ShenandoahOperation(), - _gc(gc) {}; + _gc(gc), + _do_old_gc_bootstrap(do_old_gc_bootstrap) {}; VM_Operation::VMOp_Type type() const { return VMOp_ShenandoahInitMark; } const char* name() const { return "Shenandoah Init Marking"; } virtual void doit(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp index d2742d584fb2f..d82e3e48b491d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp @@ -658,6 +658,7 @@ class VerifyThreadGCState : public ThreadClosure { }; void ShenandoahVerifier::verify_at_safepoint(const char* label, + VerifyRememberedSet remembered, VerifyForwarded forwarded, VerifyMarked marked, VerifyCollectionSet cset, VerifyLiveness liveness, VerifyRegions regions, @@ -687,6 +688,10 @@ void ShenandoahVerifier::verify_at_safepoint(const char* label, enabled = true; expected = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::EVACUATION; break; + case _verify_gcstate_updating: + enabled = true; + expected = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::UPDATEREFS; + break; case _verify_gcstate_stable: enabled = true; expected = ShenandoahHeap::STABLE; @@ -743,6 +748,12 @@ void ShenandoahVerifier::verify_at_safepoint(const char* label, if (generation != NULL) { ShenandoahHeapLocker lock(_heap->lock()); + if (remembered == _verify_remembered_for_marking) { + _heap->verify_rem_set_at_mark(); + } else if (remembered == _verify_remembered_for_updating_references) { + _heap->verify_rem_set_at_update_ref(); + } + ShenandoahCalculateRegionStatsClosure cl; generation->heap_region_iterate(&cl); size_t generation_used = generation->used(); @@ -847,6 +858,7 @@ void ShenandoahVerifier::verify_at_safepoint(const char* label, void ShenandoahVerifier::verify_generic(VerifyOption vo) { verify_at_safepoint( "Generic Verification", + _verify_remembered_disable, // do not verify remembered set _verify_forwarded_allow, // conservatively allow forwarded _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations _verify_cset_disable, // cset may be inconsistent @@ -860,6 +872,7 @@ void ShenandoahVerifier::verify_generic(VerifyOption vo) { void ShenandoahVerifier::verify_before_concmark() { verify_at_safepoint( "Before Mark", + _verify_remembered_for_marking, // verify read-only remembered set from bottom() to top() _verify_forwarded_none, // UR should have fixed up _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations _verify_cset_none, // UR should have fixed this @@ -873,6 +886,7 @@ void ShenandoahVerifier::verify_before_concmark() { void ShenandoahVerifier::verify_after_concmark() { verify_at_safepoint( "After Mark", + _verify_remembered_disable, // do not verify remembered set _verify_forwarded_none, // no forwarded references _verify_marked_complete_except_references, // bitmaps as precise as we can get, except dangling j.l.r.Refs _verify_cset_none, // no references to cset anymore @@ -891,6 +905,7 @@ void ShenandoahVerifier::verify_before_evacuation() { verify_at_safepoint( "Before Evacuation", + _verify_remembered_disable, // do not verify remembered set _verify_forwarded_none, // no forwarded references _verify_marked_complete_except_references, // walk over marked objects too _verify_cset_disable, // non-forwarded references to cset expected @@ -909,6 +924,7 @@ void ShenandoahVerifier::verify_during_evacuation() { verify_at_safepoint( "During Evacuation", + _verify_remembered_disable, // do not verify remembered set _verify_forwarded_allow, // some forwarded references are allowed _verify_marked_disable, // walk only roots _verify_cset_disable, // some cset references are not forwarded yet @@ -922,6 +938,7 @@ void ShenandoahVerifier::verify_during_evacuation() { void ShenandoahVerifier::verify_after_evacuation() { verify_at_safepoint( "After Evacuation", + _verify_remembered_disable, // do not verify remembered set _verify_forwarded_allow, // objects are still forwarded _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well _verify_cset_forwarded, // all cset refs are fully forwarded @@ -935,12 +952,13 @@ void ShenandoahVerifier::verify_after_evacuation() { void ShenandoahVerifier::verify_before_updaterefs() { verify_at_safepoint( "Before Updating References", - _verify_forwarded_allow, // forwarded references allowed - _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well - _verify_cset_forwarded, // all cset refs are fully forwarded - _verify_liveness_disable, // no reliable liveness data anymore - _verify_regions_notrash, // trash regions have been recycled already - _verify_gcstate_forwarded, // evacuation should have produced some forwarded objects + _verify_remembered_for_updating_references, // do not verify remembered set + _verify_forwarded_allow, // forwarded references allowed + _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well + _verify_cset_forwarded, // all cset refs are fully forwarded + _verify_liveness_disable, // no reliable liveness data anymore + _verify_regions_notrash, // trash regions have been recycled already + _verify_gcstate_updating, // evacuation is done, objects are forwarded, updating in process _verify_all_weak_roots ); } @@ -948,6 +966,7 @@ void ShenandoahVerifier::verify_before_updaterefs() { void ShenandoahVerifier::verify_after_updaterefs() { verify_at_safepoint( "After Updating References", + _verify_remembered_disable, // do not verify remembered set _verify_forwarded_none, // no forwarded references _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well _verify_cset_none, // no cset references, all updated @@ -961,6 +980,7 @@ void ShenandoahVerifier::verify_after_updaterefs() { void ShenandoahVerifier::verify_after_degenerated() { verify_at_safepoint( "After Degenerated GC", + _verify_remembered_disable, // do not verify remembered set _verify_forwarded_none, // all objects are non-forwarded _verify_marked_complete, // all objects are marked in complete bitmap _verify_cset_none, // no cset references @@ -974,6 +994,7 @@ void ShenandoahVerifier::verify_after_degenerated() { void ShenandoahVerifier::verify_before_fullgc() { verify_at_safepoint( "Before Full GC", + _verify_remembered_disable, // do not verify remembered set _verify_forwarded_allow, // can have forwarded objects _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations _verify_cset_disable, // cset might be foobared @@ -987,6 +1008,7 @@ void ShenandoahVerifier::verify_before_fullgc() { void ShenandoahVerifier::verify_after_fullgc() { verify_at_safepoint( "After Full GC", + _verify_remembered_disable, // do not verify remembered set _verify_forwarded_none, // all objects are non-forwarded _verify_marked_complete, // all objects are marked in complete bitmap _verify_cset_none, // no cset references diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp index a2e9eb357615d..2b6cfda73ac4c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp @@ -57,6 +57,20 @@ class ShenandoahVerifier : public CHeapObj { ShenandoahHeap* _heap; MarkBitMap* _verification_bit_map; public: + typedef enum { + // Disable remembered set verification. + _verify_remembered_disable, + + // Assure remembered set cards are dirty for every interesting pointer within + // each ShenandoahHeapRegion between bottom() and top(). This is appropriate at + // the init_mark safepoint since all TLABS are retired before we reach this code. + _verify_remembered_for_marking, + + // Assure remembered set cards are dirty for every interesting pointer within + // each ShenandoahHeapRegion between bottom() and get_update_watermark() + _verify_remembered_for_updating_references + } VerifyRememberedSet; + typedef enum { // Disable marked objects verification. _verify_marked_disable, @@ -133,7 +147,10 @@ class ShenandoahVerifier : public CHeapObj { _verify_gcstate_forwarded, // Evacuation is in progress, some objects are forwarded - _verify_gcstate_evacuation + _verify_gcstate_evacuation, + + // Evacuation is done, objects are forwarded, updating is in progress + _verify_gcstate_updating } VerifyGCState; typedef enum { @@ -167,6 +184,7 @@ class ShenandoahVerifier : public CHeapObj { private: void verify_at_safepoint(const char* label, + VerifyRememberedSet remembered, VerifyForwarded forwarded, VerifyMarked marked, VerifyCollectionSet cset, diff --git a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp index d21cdd49d46b4..aa532173e4af2 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp @@ -31,8 +31,6 @@ #include "gc/shenandoah/shenandoahYoungGeneration.hpp" #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" -#undef TRACE_PROMOTION - ShenandoahYoungGeneration::ShenandoahYoungGeneration(uint max_queues, size_t max_capacity, size_t soft_max_capacity) : ShenandoahGeneration(YOUNG, max_queues, max_capacity, soft_max_capacity), _old_gen_task_queues(nullptr) { @@ -69,9 +67,10 @@ class ShenandoahPromoteTenuredRegionsTask : public AbstractGangTask { ShenandoahHeapRegion* r = _regions->next(); while (r != NULL) { if (r->is_young() && r->age() >= InitialTenuringThreshold && ((r->is_regular() && !r->has_young_lab_flag()) || r->is_humongous_start())) { - // The above condition filtered out humongous continuations, among other states. - // Here we rely on promote() below promoting related continuation regions when encountering a homongous start. - size_t promoted = r->promote(); + // The thread that first encounters a humongous start region promotes the associated humonogous continuations, + // so we do not process humongous continuations directly. Below, we rely on promote() to promote related + // continuation regions when encountering a homongous start. + size_t promoted = r->promote(false); Atomic::add(&_promoted, promoted); } r = _regions->next(); @@ -95,24 +94,20 @@ void ShenandoahYoungGeneration::promote_all_regions() { for (size_t index = 0; index < heap->num_regions(); index++) { ShenandoahHeapRegion* r = heap->get_region(index); if (r->is_young()) { - r->promote(); + r->promote(true); } } assert(_affiliated_region_count == 0, "young generation must not have affiliated regions after reset"); _used = 0; - - // HEY! Better to use a service of ShenandoahScanRemembered for the following. - - // We can clear the entire card table here because we've just promoted all - // young regions to old, so there can be no old->young pointers at this point. - ShenandoahBarrierSet::barrier_set()->card_table()->clear(); } bool ShenandoahYoungGeneration::contains(ShenandoahHeapRegion* region) const { + // TODO: why not test for equals YOUNG_GENERATION? As written, returns true for regions that are FREE return region->affiliation() != OLD_GENERATION; } void ShenandoahYoungGeneration::parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl) { + // Just iterate over the young generation here. ShenandoahGenerationRegionClosure young_regions(cl); ShenandoahHeap::heap()->parallel_heap_region_iterate(&young_regions); } From 5ebe8d10b61e783807648e827a87f0d3a5126cfe Mon Sep 17 00:00:00 2001 From: William Kemper Date: Fri, 6 Aug 2021 16:20:13 +0000 Subject: [PATCH 051/254] Check that pointer is in heap before checking the affiliation of owning region Reviewed-by: rkennke --- src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp | 4 ++-- .../share/gc/shenandoah/shenandoahReferenceProcessor.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index c8e2e93193e2a..9a835e7e7b633 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -753,11 +753,11 @@ bool ShenandoahHeap::is_in(const void* p) const { } bool ShenandoahHeap::is_in_young(const void* p) const { - return heap_region_containing(p)->affiliation() == ShenandoahRegionAffiliation::YOUNG_GENERATION; + return is_in(p) && heap_region_containing(p)->affiliation() == ShenandoahRegionAffiliation::YOUNG_GENERATION; } bool ShenandoahHeap::is_in_old(const void* p) const { - return heap_region_containing(p)->affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION; + return is_in(p) && heap_region_containing(p)->affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION; } void ShenandoahHeap::op_uncommit(double shrink_before, size_t shrink_until) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp b/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp index 99c410dad2ca7..205ac7c82a101 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp @@ -398,7 +398,7 @@ oop ShenandoahReferenceProcessor::drop(oop reference, ReferenceType type) { // the cycle, we need to dirty the card if the reference is old and the referent is young. Note // that if the reference is not dropped, then its pointer to the referent will be nulled before // evacuation begins so card does not need to be dirtied. - if (heap->mode()->is_generational() && heap->is_old(reference) && heap->is_in_young(referent)) { + if (heap->mode()->is_generational() && heap->is_in_old(reference) && heap->is_in_young(referent)) { // Note: would be sufficient to mark only the card that holds the start of this Reference object. heap->card_scan()->mark_range_as_dirty(cast_from_oop(reference), reference->size()); } From be67d449b903e28b47771e12c6f01c2e15693140 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Mon, 9 Aug 2021 16:07:23 +0000 Subject: [PATCH 052/254] Generational support for weak roots and references Reviewed-by: rkennke --- .../mode/shenandoahGenerationalMode.cpp | 13 +++++-- .../mode/shenandoahGenerationalMode.hpp | 4 ++- .../shenandoahBarrierSet.inline.hpp | 2 ++ .../gc/shenandoah/shenandoahConcurrentGC.cpp | 15 +++++--- .../shenandoah/shenandoahConcurrentMark.cpp | 6 ++-- .../share/gc/shenandoah/shenandoahFreeSet.cpp | 5 +-- .../share/gc/shenandoah/shenandoahFullGC.cpp | 4 +-- .../gc/shenandoah/shenandoahGeneration.cpp | 14 +++++--- .../gc/shenandoah/shenandoahGeneration.hpp | 10 +++++- .../shenandoah/shenandoahGlobalGeneration.hpp | 2 ++ .../share/gc/shenandoah/shenandoahHeap.cpp | 21 ++++++++--- .../share/gc/shenandoah/shenandoahHeap.hpp | 12 ++----- .../gc/shenandoah/shenandoahHeap.inline.hpp | 1 - .../gc/shenandoah/shenandoahHeapRegion.cpp | 2 ++ .../share/gc/shenandoah/shenandoahMark.cpp | 3 +- .../share/gc/shenandoah/shenandoahOldGC.cpp | 36 ++++++++++--------- .../gc/shenandoah/shenandoahOldGeneration.cpp | 10 +++++- .../gc/shenandoah/shenandoahOldGeneration.hpp | 15 ++++---- .../shenandoahReferenceProcessor.cpp | 10 +++++- .../share/gc/shenandoah/shenandoahSTWMark.cpp | 4 +-- .../share/gc/shenandoah/shenandoahUtils.cpp | 2 +- .../shenandoah/shenandoahYoungGeneration.cpp | 4 +++ .../shenandoah/shenandoahYoungGeneration.hpp | 17 ++++----- 23 files changed, 138 insertions(+), 74 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp index 1430e815b3e3b..61ee91072ef70 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp @@ -59,7 +59,15 @@ void ShenandoahGenerationalMode::initialize_flags() const { SHENANDOAH_CHECK_FLAG_UNSET(ClassUnloading); } -const char *affiliation_name(ShenandoahRegionAffiliation type) { +const char* affiliation_name(oop ptr) { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + assert(heap->is_in(ptr), "Oop must be in the heap."); + ShenandoahHeapRegion* region = heap->heap_region_containing(ptr); + return affiliation_name(region->affiliation()); +} + + +const char* affiliation_name(ShenandoahRegionAffiliation type) { switch (type) { case ShenandoahRegionAffiliation::FREE: return "FREE"; @@ -68,6 +76,7 @@ const char *affiliation_name(ShenandoahRegionAffiliation type) { case ShenandoahRegionAffiliation::OLD_GENERATION: return "OLD"; default: - return "UnrecognizedAffiliation"; + ShouldNotReachHere(); + return nullptr; } } diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp index 377e7eef40a2d..655e2b9fac5a2 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp @@ -26,6 +26,7 @@ #define SHARE_GC_SHENANDOAH_MODE_SHENANDOAHGENERATIONALMODE_HPP #include "gc/shenandoah/mode/shenandoahMode.hpp" +#include "oops/oopsHierarchy.hpp" enum GenerationMode { YOUNG, @@ -39,7 +40,8 @@ enum ShenandoahRegionAffiliation { OLD_GENERATION }; -const char *affiliation_name(ShenandoahRegionAffiliation type); +const char* affiliation_name(oop ptr); +const char* affiliation_name(ShenandoahRegionAffiliation type); class ShenandoahGenerationalMode : public ShenandoahMode { public: diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp index b92fe6d9407a8..c437b0ea802e8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp @@ -107,6 +107,7 @@ inline oop ShenandoahBarrierSet::load_reference_barrier(oop obj, T* load_addr) { // Prevent resurrection of unreachable phantom (i.e. weak-native) references. if (HasDecorator::value && obj != NULL && _heap->is_concurrent_weak_root_in_progress() && + _heap->is_in_active_generation(obj) && !_heap->marking_context()->is_marked(obj)) { return NULL; } @@ -114,6 +115,7 @@ inline oop ShenandoahBarrierSet::load_reference_barrier(oop obj, T* load_addr) { // Prevent resurrection of unreachable weak references. if ((HasDecorator::value || HasDecorator::value) && obj != NULL && _heap->is_concurrent_weak_root_in_progress() && + _heap->is_in_active_generation(obj) && !_heap->marking_context()->is_marked_strong(obj)) { return NULL; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index 48aac46e2fb5e..99eb396dca7d8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -507,7 +507,7 @@ void ShenandoahConcurrentGC::op_init_mark() { } // Weak reference processing - ShenandoahReferenceProcessor* rp = heap->ref_processor(); + ShenandoahReferenceProcessor* rp = _generation->ref_processor(); rp->reset_thread_locals(); rp->set_soft_reference_policy(heap->soft_ref_policy()->should_clear_all_soft_refs()); @@ -644,7 +644,7 @@ void ShenandoahConcurrentGC::op_weak_refs() { assert(heap->is_concurrent_weak_root_in_progress(), "Only during this phase"); // Concurrent weak refs processing ShenandoahGCWorkerPhase worker_phase(ShenandoahPhaseTimings::conc_weak_refs); - heap->ref_processor()->process_references(ShenandoahPhaseTimings::conc_weak_refs, heap->workers(), true /* concurrent */); + _generation->ref_processor()->process_references(ShenandoahPhaseTimings::conc_weak_refs, heap->workers(), true /* concurrent */); } class ShenandoahEvacUpdateCleanupOopStorageRootsClosure : public BasicOopIterateClosure { @@ -671,8 +671,15 @@ void ShenandoahEvacUpdateCleanupOopStorageRootsClosure::do_oop(oop* p) { const oop obj = RawAccess<>::oop_load(p); if (!CompressedOops::is_null(obj)) { if (!_mark_context->is_marked(obj)) { - shenandoah_assert_correct(p, obj); - Atomic::cmpxchg(p, obj, oop(NULL)); + if (_heap->is_in_active_generation(obj)) { + // TODO: This worries me. Here we are asserting that an unmarked from-space object is 'correct'. + // Normally, I would call this a bogus assert, but there seems to be a legitimate use-case for + // accessing from-space objects during class unloading. However, the from-space object may have + // been "filled". We've made no effort to prevent old generation classes being unloaded by young + // gen (and vice-versa). + shenandoah_assert_correct(p, obj); + Atomic::cmpxchg(p, obj, oop(NULL)); + } } else if (_evac_in_progress && _heap->in_collection_set(obj)) { oop resolved = ShenandoahBarrierSet::resolve_forwarded_not_null(obj); if (resolved == obj) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp index 525126bfb3e53..0e46aa04810c2 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp @@ -86,7 +86,7 @@ class ShenandoahConcurrentMarkingTask : public AbstractGangTask { ShenandoahHeap* heap = ShenandoahHeap::heap(); ShenandoahConcurrentWorkerSession worker_session(worker_id); ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers); - ShenandoahReferenceProcessor* rp = heap->ref_processor(); + ShenandoahReferenceProcessor* rp = heap->active_generation()->ref_processor(); assert(rp != NULL, "need reference processor"); _cm->mark_loop(GENERATION, worker_id, _terminator, rp, true, // cancellable @@ -136,7 +136,7 @@ class ShenandoahFinalMarkingTask : public AbstractGangTask { ShenandoahHeap* heap = ShenandoahHeap::heap(); ShenandoahParallelWorkerSession worker_session(worker_id); - ShenandoahReferenceProcessor* rp = heap->ref_processor(); + ShenandoahReferenceProcessor* rp = heap->active_generation()->ref_processor(); // First drain remaining SATB buffers. { @@ -214,7 +214,7 @@ void ShenandoahConcurrentMark::mark_concurrent_roots() { TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats()); WorkGang* workers = heap->workers(); - ShenandoahReferenceProcessor* rp = heap->ref_processor(); + ShenandoahReferenceProcessor* rp = _generation->ref_processor(); _generation->reserve_task_queues(workers->active_workers()); switch (_generation->generation_mode()) { case YOUNG: { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index 63b29d13ef64c..5c9874e607b3f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -168,10 +168,7 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah if (r->affiliation() == ShenandoahRegionAffiliation::FREE) { ShenandoahMarkingContext* const ctx = _heap->complete_marking_context(); - if (req.affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) { - // This free region might have garbage in its remembered set representation. - _heap->clear_cards_for(r); - } + r->set_affiliation(req.affiliation()); r->set_update_watermark(r->bottom()); ctx->capture_top_at_mark_start(r); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index 00c71c52088cc..7466b8b6164c4 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -184,7 +184,7 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) { assert(!heap->global_generation()->is_mark_complete(), "sanity"); // e. Abandon reference discovery and clear all discovered references. - ShenandoahReferenceProcessor* rp = heap->ref_processor(); + ShenandoahReferenceProcessor* rp = heap->global_generation()->ref_processor(); rp->abandon_partial_discovery(); // f. Set back forwarded objects bit back, in case some steps above dropped it. @@ -299,7 +299,7 @@ void ShenandoahFullGC::phase1_mark_heap() { heap->set_unload_classes(heap->global_generation()->heuristics()->can_unload_classes()); - ShenandoahReferenceProcessor* rp = heap->ref_processor(); + ShenandoahReferenceProcessor* rp = heap->global_generation()->ref_processor(); // enable ("weak") refs discovery rp->set_soft_reference_policy(true); // forcefully purge all soft references diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index 7f1e30e44d639..a0341f76b2d91 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -29,6 +29,7 @@ #include "gc/shenandoah/shenandoahHeap.hpp" #include "gc/shenandoah/shenandoahMarkClosures.hpp" #include "gc/shenandoah/shenandoahMonitoringSupport.hpp" +#include "gc/shenandoah/shenandoahReferenceProcessor.hpp" #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp" #include "gc/shenandoah/shenandoahUtils.hpp" #include "gc/shenandoah/shenandoahVerifier.hpp" @@ -259,19 +260,22 @@ void ShenandoahGeneration::cancel_marking() { } set_mark_incomplete(); _task_queues->clear(); + + ref_processor()->abandon_partial_discovery(); } ShenandoahGeneration::ShenandoahGeneration(GenerationMode generation_mode, - uint max_queues, + uint max_workers, size_t max_capacity, size_t soft_max_capacity) : _generation_mode(generation_mode), - _task_queues(new ShenandoahObjToScanQueueSet(max_queues)), + _task_queues(new ShenandoahObjToScanQueueSet(max_workers)), + _ref_processor(new ShenandoahReferenceProcessor(MAX2(max_workers, 1U))), _affiliated_region_count(0), _used(0), _max_capacity(max_capacity), _soft_max_capacity(soft_max_capacity) { _is_marking_complete.set(); - assert(max_queues > 0, "At least one queue"); - for (uint i = 0; i < max_queues; ++i) { + assert(max_workers > 0, "At least one queue"); + for (uint i = 0; i < max_workers; ++i) { ShenandoahObjToScanQueue* task_queue = new ShenandoahObjToScanQueue(); task_queue->initialize(); _task_queues->register_queue(i, task_queue); @@ -302,7 +306,7 @@ void ShenandoahGeneration::scan_remembered_set() { reserve_task_queues(nworkers); ShenandoahConcurrentPhase gc_phase("Concurrent remembered set scanning", ShenandoahPhaseTimings::init_scan_rset); - ShenandoahReferenceProcessor* rp = heap->ref_processor(); + ShenandoahReferenceProcessor* rp = ref_processor(); ShenandoahRegionIterator regions; ShenandoahScanRememberedTask task(task_queues(), old_gen_task_queues(), rp, ®ions); heap->workers()->run_task(&task); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index 4cd218055b944..730ef46bfe9b5 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -33,6 +33,7 @@ class ShenandoahHeapRegion; class ShenandoahHeapRegionClosure; +class ShenandoahReferenceProcessor; class ShenandoahGeneration : public CHeapObj { private: @@ -43,6 +44,8 @@ class ShenandoahGeneration : public CHeapObj { ShenandoahObjToScanQueueSet* _task_queues; ShenandoahSharedFlag _is_marking_complete; + ShenandoahReferenceProcessor* const _ref_processor; + protected: // Usage size_t _affiliated_region_count; @@ -51,13 +54,15 @@ class ShenandoahGeneration : public CHeapObj { size_t _soft_max_capacity; public: - ShenandoahGeneration(GenerationMode generation_mode, uint max_queues, size_t max_capacity, size_t soft_max_capacity); + ShenandoahGeneration(GenerationMode generation_mode, uint max_workers, size_t max_capacity, size_t soft_max_capacity); ~ShenandoahGeneration(); inline GenerationMode generation_mode() const { return _generation_mode; } inline ShenandoahHeuristics* heuristics() const { return _heuristics; } + ShenandoahReferenceProcessor* ref_processor() { return _ref_processor; } + virtual const char* name() const = 0; ShenandoahHeuristics* initialize_heuristics(ShenandoahMode* gc_mode); @@ -96,6 +101,9 @@ class ShenandoahGeneration : public CHeapObj { // Return true if this region is affiliated with this generation. virtual bool contains(ShenandoahHeapRegion* region) const = 0; + // Return true if this object is affiliated with this generation. + virtual bool contains(oop obj) const = 0; + // Apply closure to all regions affiliated with this generation. virtual void parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl) = 0; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp index 0f70f3ce04ac9..cd4e50ddfd026 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp @@ -48,6 +48,8 @@ class ShenandoahGlobalGeneration : public ShenandoahGeneration { bool contains(ShenandoahHeapRegion* region) const; + bool contains(oop obj) const override { return true; } + void parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl); void heap_region_iterate(ShenandoahHeapRegionClosure* cl); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 9a835e7e7b633..e40657c4b8bf9 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -526,7 +526,6 @@ ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) : _gc_timer(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer()), _soft_ref_policy(), _log_min_obj_alignment_in_bytes(LogMinObjAlignmentInBytes), - _ref_processor(new ShenandoahReferenceProcessor(MAX2(_max_workers, 1U))), _marking_context(NULL), _bitmap_size(0), _bitmap_regions_per_slice(0), @@ -760,6 +759,21 @@ bool ShenandoahHeap::is_in_old(const void* p) const { return is_in(p) && heap_region_containing(p)->affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION; } +bool ShenandoahHeap::is_in_active_generation(oop obj) const { + if (!mode()->is_generational()) { + // everything is the same single generation + return true; + } + + if (active_generation() == NULL) { + // no collection is happening, only expect this to be called + // when concurrent processing is active, but that could change + return false; + } + + return active_generation()->contains(obj); +} + void ShenandoahHeap::op_uncommit(double shrink_before, size_t shrink_until) { assert (ShenandoahUncommit, "should be enabled"); @@ -1742,7 +1756,7 @@ void ShenandoahHeap::stw_weak_refs(bool full_gc) { : ShenandoahPhaseTimings::degen_gc_weakrefs; ShenandoahTimingsTracker t(phase); ShenandoahGCWorkerPhase worker_phase(phase); - ref_processor()->process_references(phase, workers(), false /* concurrent */); + active_generation()->ref_processor()->process_references(phase, workers(), false /* concurrent */); } void ShenandoahHeap::prepare_update_heap_references(bool concurrent) { @@ -1861,9 +1875,6 @@ void ShenandoahHeap::cancel_concurrent_mark() { _global_generation->cancel_marking(); ShenandoahBarrierSet::satb_mark_queue_set().abandon_partial_marking(); - - // HEY! Previously, only ShenandoahConcurrentMark::cancel (static) cleared ref processor. - ref_processor()->abandon_partial_discovery(); } void ShenandoahHeap::cancel_gc(GCCause::Cause cause) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index 82b7a0acf9073..e25a0da2cd064 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -140,6 +140,7 @@ class ShenandoahHeap : public CollectedHeap { friend class ShenandoahSafepoint; // Supported GC friend class ShenandoahConcurrentGC; + friend class ShenandoahOldGC; friend class ShenandoahDegenGC; friend class ShenandoahFullGC; @@ -156,7 +157,7 @@ class ShenandoahHeap : public CollectedHeap { return &_lock; } - ShenandoahGeneration* active_generation() { + ShenandoahGeneration* active_generation() const { // last or latest generation might be a better name here. return _gc_generation; } @@ -482,14 +483,6 @@ class ShenandoahHeap : public CollectedHeap { GCTracer* tracer(); ConcurrentGCTimer* gc_timer() const; -// ---------- Reference processing -// -private: - ShenandoahReferenceProcessor* const _ref_processor; - -public: - ShenandoahReferenceProcessor* ref_processor() { return _ref_processor; } - // ---------- Class Unloading // private: @@ -524,6 +517,7 @@ class ShenandoahHeap : public CollectedHeap { bool is_in(const void* p) const; + bool is_in_active_generation(oop obj) const; bool is_in_young(const void* p) const; bool is_in_old(const void* p) const; inline bool is_old(oop pobj) const; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp index 95c00e15de40b..3aac49352c7cc 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp @@ -404,7 +404,6 @@ void ShenandoahHeap::increase_object_age(oop obj, uint additional_age) { } } - inline bool ShenandoahHeap::is_old(oop obj) const { return is_gc_generation_young() && is_in_old(obj); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index 857f9e189aadf..d9e2700edbc45 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -610,6 +610,8 @@ void ShenandoahHeapRegion::recycle() { make_empty(); set_affiliation(FREE); + heap->clear_cards_for(this); + if (ZapUnusedHeapArea) { SpaceMangler::mangle_region(MemRegion(bottom(), end())); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp index 266227606d6c6..034153fd552b3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp @@ -110,7 +110,8 @@ void ShenandoahMark::mark_loop_work(T* cl, ShenandoahLiveData* live_data, uint w ShenandoahObjToScanQueue* q; ShenandoahMarkTask t; - heap->ref_processor()->set_mark_closure(worker_id, cl); + assert(heap->active_generation()->generation_mode() == GENERATION, "Sanity"); + heap->active_generation()->ref_processor()->set_mark_closure(worker_id, cl); /* * Process outstanding queues, if any. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp index a533ccb0883f7..a4ce1da13691d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp @@ -32,6 +32,7 @@ #include "gc/shenandoah/shenandoahGeneration.hpp" #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" #include "gc/shenandoah/shenandoahWorkerPolicy.hpp" +#include "prims/jvmtiTagMap.hpp" #include "utilities/events.hpp" class ShenandoahConcurrentCoalesceAndFillTask : public AbstractGangTask { @@ -72,7 +73,6 @@ ShenandoahOldGC::ShenandoahOldGC(ShenandoahGeneration* generation, ShenandoahSha void ShenandoahOldGC::entry_old_evacuations() { ShenandoahHeap* heap = ShenandoahHeap::heap(); ShenandoahOldHeuristics* old_heuristics = heap->old_heuristics(); - entry_coalesce_and_fill(); old_heuristics->start_old_evacuations(); } @@ -94,9 +94,8 @@ void ShenandoahOldGC::op_final_mark() { _mark.finish_mark(); assert(!heap->cancelled_gc(), "STW mark cannot OOM"); - // Believe notifying JVMTI that the tagmap table will need cleaning is not relevant following old-gen mark - // so commenting out for now: - // JvmtiTagMap::set_needs_cleaning(); + // We need to do this because weak root cleaning reports the number of dead handles + JvmtiTagMap::set_needs_cleaning(); { ShenandoahGCPhase phase(ShenandoahPhaseTimings::choose_cset); @@ -105,6 +104,9 @@ void ShenandoahOldGC::op_final_mark() { _generation->heuristics()->choose_collection_set(nullptr, nullptr); } + heap->set_unload_classes(false); + heap->prepare_concurrent_roots(); + // Believe verification following old-gen concurrent mark needs to be different than verification following // young-gen concurrent mark, so am commenting this out for now: // if (ShenandoahVerify) { @@ -129,21 +131,15 @@ bool ShenandoahOldGC::collect(GCCause::Cause cause) { // Complete marking under STW vmop_entry_final_mark(); - entry_old_evacuations(); - // We aren't dealing with old generation evacuation yet. Our heuristic // should not have built a cset in final mark. assert(!heap->is_evacuation_in_progress(), "Old gen evacuations are not supported"); - // Concurrent stack processing - if (heap->is_evacuation_in_progress()) { - entry_thread_roots(); - } - // Process weak roots that might still point to regions that would be broken by cleanup if (heap->is_concurrent_weak_root_in_progress()) { entry_weak_refs(); entry_weak_roots(); + heap->set_concurrent_weak_root_in_progress(false); } // Final mark might have reclaimed some immediate garbage, kick cleanup to reclaim @@ -161,12 +157,18 @@ bool ShenandoahOldGC::collect(GCCause::Cause cause) { entry_class_unloading(); } - // Processing strong roots - // This may be skipped if there is nothing to update/evacuate. - // If so, strong_root_in_progress would be unset. - if (heap->is_concurrent_strong_root_in_progress()) { - entry_strong_roots(); - } + // Coalesce and fill objects _after_ weak root processing and class unloading. + // Weak root and reference processing makes assertions about unmarked referents + // that will fail if they've been overwritten with filler objects. There is also + // a case in the LRB that permits access to from-space objects for the purpose + // of class unloading that is unlikely to function correctly if the object has + // been filled. + entry_coalesce_and_fill(); + + // Prepare for old evacuations (actual evacuations will happen on subsequent young collects). + entry_old_evacuations(); + + assert(!heap->is_concurrent_strong_root_in_progress(), "No evacuations during old gc."); entry_rendezvous_roots(); return true; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp index 5329def70f595..99b23e867717c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp @@ -32,6 +32,7 @@ #include "gc/shenandoah/shenandoahMark.inline.hpp" #include "gc/shenandoah/shenandoahOldGeneration.hpp" #include "gc/shenandoah/shenandoahOopClosures.inline.hpp" +#include "gc/shenandoah/shenandoahReferenceProcessor.hpp" #include "gc/shenandoah/shenandoahStringDedup.hpp" #include "gc/shenandoah/shenandoahUtils.hpp" @@ -128,7 +129,10 @@ class ShenandoahPurgeSATBTask : public AbstractGangTask { }; ShenandoahOldGeneration::ShenandoahOldGeneration(uint max_queues, size_t max_capacity, size_t soft_max_capacity) - : ShenandoahGeneration(OLD, max_queues, max_capacity, soft_max_capacity) {} + : ShenandoahGeneration(OLD, max_queues, max_capacity, soft_max_capacity) { + // Always clear references for old generation + ref_processor()->set_soft_reference_policy(true); +} const char* ShenandoahOldGeneration::name() const { return "OLD"; @@ -171,3 +175,7 @@ void ShenandoahOldGeneration::purge_satb_buffers(bool abandon) { heap->workers()->run_task(&purge_satb_task); } } + +bool ShenandoahOldGeneration::contains(oop obj) const { + return ShenandoahHeap::heap()->is_in_old(obj); +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp index c6a7bdb9a49b0..f2f39b79cdfdd 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp @@ -34,14 +34,17 @@ class ShenandoahOldGeneration : public ShenandoahGeneration { public: ShenandoahOldGeneration(uint max_queues, size_t max_capacity, size_t soft_max_capacity); - const char* name() const; + const char* name() const override; - bool contains(ShenandoahHeapRegion* region) const; - void parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl); + bool contains(ShenandoahHeapRegion* region) const override; - void heap_region_iterate(ShenandoahHeapRegionClosure* cl); + bool contains(oop obj) const override; - void set_concurrent_mark_in_progress(bool in_progress); + void parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl) override; + + void heap_region_iterate(ShenandoahHeapRegionClosure* cl) override; + + void set_concurrent_mark_in_progress(bool in_progress) override; // We leave the SATB barrier on for the entirety of the old generation // marking phase. In some cases, this can cause a write to a perfectly @@ -63,7 +66,7 @@ class ShenandoahOldGeneration : public ShenandoahGeneration { // the performance impact would be too severe. void purge_satb_buffers(bool abandon); protected: - bool is_concurrent_mark_in_progress(); + bool is_concurrent_mark_in_progress() override; }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp b/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp index 205ac7c82a101..32ed6fc8f1ced 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp @@ -25,6 +25,7 @@ #include "precompiled.hpp" #include "classfile/javaClasses.hpp" +#include "gc/shenandoah/mode/shenandoahGenerationalMode.hpp" #include "gc/shenandoah/shenandoahOopClosures.inline.hpp" #include "gc/shenandoah/shenandoahReferenceProcessor.hpp" #include "gc/shenandoah/shenandoahThreadLocalData.hpp" @@ -267,6 +268,7 @@ bool ShenandoahReferenceProcessor::should_discover(oop reference, ReferenceType T* referent_addr = (T*) java_lang_ref_Reference::referent_addr_raw(reference); T heap_oop = RawAccess<>::oop_load(referent_addr); oop referent = CompressedOops::decode(heap_oop); + ShenandoahHeap* heap = ShenandoahHeap::heap(); if (is_inactive(reference, referent, type)) { log_trace(gc,ref)("Reference inactive: " PTR_FORMAT, p2i(reference)); @@ -283,6 +285,11 @@ bool ShenandoahReferenceProcessor::should_discover(oop reference, ReferenceType return false; } + if (!heap->is_in_active_generation(referent)) { + log_trace(gc,ref)("Referent outside of active generation: " PTR_FORMAT, p2i(referent)); + return false; + } + return true; } @@ -371,7 +378,8 @@ bool ShenandoahReferenceProcessor::discover_reference(oop reference, ReferenceTy return false; } - log_trace(gc, ref)("Encountered Reference: " PTR_FORMAT " (%s)", p2i(reference), reference_type_name(type)); + log_trace(gc, ref)("Encountered Reference: " PTR_FORMAT " (%s, %s)", + p2i(reference), reference_type_name(type), affiliation_name(reference)); uint worker_id = ShenandoahThreadLocalData::worker_id(Thread::current()); _ref_proc_thread_locals->inc_encountered(type); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp index b8341dd5e29f1..90c0cc7b9469f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp @@ -68,7 +68,7 @@ ShenandoahSTWMark::ShenandoahSTWMark(ShenandoahGeneration* generation, bool full void ShenandoahSTWMark::mark() { // Weak reference processing ShenandoahHeap* const heap = ShenandoahHeap::heap(); - ShenandoahReferenceProcessor* rp = heap->ref_processor(); + ShenandoahReferenceProcessor* rp = heap->active_generation()->ref_processor(); rp->reset_thread_locals(); rp->set_soft_reference_policy(heap->soft_ref_policy()->should_clear_all_soft_refs()); @@ -120,7 +120,7 @@ void ShenandoahSTWMark::mark_roots(uint worker_id) { void ShenandoahSTWMark::finish_mark(uint worker_id) { ShenandoahPhaseTimings::Phase phase = _full_gc ? ShenandoahPhaseTimings::full_gc_mark : ShenandoahPhaseTimings::degen_gc_stw_mark; ShenandoahWorkerTimingsTracker timer(phase, ShenandoahPhaseTimings::ParallelMark, worker_id); - ShenandoahReferenceProcessor* rp = ShenandoahHeap::heap()->ref_processor(); + ShenandoahReferenceProcessor* rp = ShenandoahHeap::heap()->active_generation()->ref_processor(); mark_loop(_generation->generation_mode(), worker_id, &_terminator, rp, diff --git a/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp b/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp index 869a1e255169e..6a9b0a1759fbc 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp @@ -70,7 +70,7 @@ ShenandoahGCSession::~ShenandoahGCSession() { _generation->heuristics()->record_cycle_end(); _timer->register_gc_end(); _heap->trace_heap_after_gc(_tracer); - _tracer->report_gc_reference_stats(_heap->ref_processor()->reference_process_stats()); + _tracer->report_gc_reference_stats(_generation->ref_processor()->reference_process_stats()); _tracer->report_gc_end(_timer->gc_end(), _timer->time_partitions()); assert(!ShenandoahGCPhase::is_current_phase_valid(), "No current GC phase"); _heap->set_gc_cause(GCCause::_no_gc); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp index aa532173e4af2..9d7ee3810ba96 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp @@ -127,3 +127,7 @@ void ShenandoahYoungGeneration::reserve_task_queues(uint workers) { _old_gen_task_queues->reserve(workers); } } + +bool ShenandoahYoungGeneration::contains(oop obj) const { + return ShenandoahHeap::heap()->is_in_young(obj); +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp index 1f12df47169fa..6241f7f4cfb5c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp @@ -34,14 +34,15 @@ class ShenandoahYoungGeneration : public ShenandoahGeneration { public: ShenandoahYoungGeneration(uint max_queues, size_t max_capacity, size_t max_soft_capacity); - virtual const char* name() const; + const char* name() const override; - virtual void set_concurrent_mark_in_progress(bool in_progress); - virtual void parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl); + void set_concurrent_mark_in_progress(bool in_progress) override; + void parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl) override; - void heap_region_iterate(ShenandoahHeapRegionClosure* cl); + void heap_region_iterate(ShenandoahHeapRegionClosure* cl) override; - bool contains(ShenandoahHeapRegion* region) const; + bool contains(ShenandoahHeapRegion* region) const override; + bool contains(oop obj) const override; void promote_tenured_regions(); void promote_all_regions(); @@ -50,14 +51,14 @@ class ShenandoahYoungGeneration : public ShenandoahGeneration { _old_gen_task_queues = old_gen_queues; } - ShenandoahObjToScanQueueSet* old_gen_task_queues() const { + ShenandoahObjToScanQueueSet* old_gen_task_queues() const override { return _old_gen_task_queues; } - virtual void reserve_task_queues(uint workers); + void reserve_task_queues(uint workers) override; protected: - bool is_concurrent_mark_in_progress(); + bool is_concurrent_mark_in_progress() override; }; #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHYOUNGGENERATION_HPP From 6ee7458222bf92540867d66ef6d41f58ac0f1137 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Tue, 10 Aug 2021 17:27:21 +0000 Subject: [PATCH 053/254] Enable skip update references Reviewed-by: rkennke --- .../share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp | 2 +- src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp index 1b4c28354c028..336a3c5a0ca4a 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp @@ -114,7 +114,7 @@ bool ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec free_regions++; free += ShenandoahHeapRegion::region_size_bytes(); } else if (region->is_regular()) { - if (!region->has_live() && !heap->mode()->is_generational()) { + if (!region->has_live()) { // We can recycle it right away and put it in the free set. immediate_regions++; immediate_garbage += garbage; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index e40657c4b8bf9..22a1e22b82b57 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -1152,8 +1152,7 @@ class ShenandoahEvacuationTask : public AbstractGangTask { ShenandoahConcurrentEvacuateRegionObjectClosure cl(_sh); ShenandoahHeapRegion* r; while ((r =_cs->claim_next()) != NULL) { - // Generational mode doesn't support immediate collection - assert(_sh->mode()->is_generational() || r->has_live(), "Region " SIZE_FORMAT " should have been reclaimed early", r->index()); + assert(r->has_live(), "Region " SIZE_FORMAT " should have been reclaimed early", r->index()); _sh->marked_object_iterate(r, &cl); if (ShenandoahPacing) { From b96a883fb138c410900beb773dd7b0593dcd53e2 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Tue, 10 Aug 2021 17:28:20 +0000 Subject: [PATCH 054/254] Re-enable class unloading Reviewed-by: rkennke --- .../gc/shenandoah/mode/shenandoahGenerationalMode.cpp | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp index 61ee91072ef70..726cb4ed10a95 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp @@ -30,14 +30,6 @@ #include "runtime/globals_extension.hpp" void ShenandoahGenerationalMode::initialize_flags() const { - // When we fill in dead objects during update refs, we use oop::size, - // which depends on the klass being loaded. However, if these dead objects - // were the last referrers to the klass, it will be unloaded and we'll - // crash. Class unloading is disabled until we're able to sort this out. - FLAG_SET_ERGO(ClassUnloading, false); - FLAG_SET_ERGO(ClassUnloadingWithConcurrentMark, false); - FLAG_SET_ERGO(ShenandoahUnloadClassesFrequency, 0); - if (ClassUnloading) { // Leaving this here for the day we re-enable class unloading FLAG_SET_DEFAULT(ShenandoahSuspendibleWorkers, true); @@ -56,7 +48,6 @@ void ShenandoahGenerationalMode::initialize_flags() const { SHENANDOAH_CHECK_FLAG_SET(ShenandoahSATBBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahCASBarrier); SHENANDOAH_CHECK_FLAG_SET(ShenandoahCloneBarrier); - SHENANDOAH_CHECK_FLAG_UNSET(ClassUnloading); } const char* affiliation_name(oop ptr) { From df8c05626d465451c511bdc8f2c664cf7696676f Mon Sep 17 00:00:00 2001 From: William Kemper Date: Mon, 16 Aug 2021 17:11:27 +0000 Subject: [PATCH 055/254] Make immediate trash of old regions with no live objects Reviewed-by: rkennke --- .../shenandoah/heuristics/shenandoahOldHeuristics.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp index c9ab820507acb..d4d4aac603bdf 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp @@ -171,9 +171,13 @@ void ShenandoahOldHeuristics::prepare_for_old_collections() { total_garbage += garbage; if (region->is_regular()) { - candidates[cand_idx]._region = region; - candidates[cand_idx]._garbage = garbage; - cand_idx++; + if (!region->has_live()) { + region->make_trash_immediate(); + } else { + candidates[cand_idx]._region = region; + candidates[cand_idx]._garbage = garbage; + cand_idx++; + } } else if (region->is_humongous_start()) { if (!region->has_live()) { // The humongous object is dead, we can just return this region and the continuations From c02907e53ea63fa7fe30961c3d917d7a5874f30b Mon Sep 17 00:00:00 2001 From: Aleksey Shipilev Date: Thu, 19 Aug 2021 10:58:41 +0000 Subject: [PATCH 056/254] Fix Zero builds Reviewed-by: rkennke --- .../share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp index 726cb4ed10a95..2d7d4daee6a4d 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp @@ -25,6 +25,7 @@ #include "precompiled.hpp" #include "gc/shenandoah/mode/shenandoahGenerationalMode.hpp" #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" +#include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "logging/log.hpp" #include "logging/logTag.hpp" #include "runtime/globals_extension.hpp" From 7dc75373a09d3ca7d0a01ab7f5ece8696590ad6e Mon Sep 17 00:00:00 2001 From: William Kemper Date: Fri, 20 Aug 2021 21:16:54 +0000 Subject: [PATCH 057/254] Update alloc liveness data during final mark of old generation Reviewed-by: rkennke --- .../gc/shenandoah/shenandoahGeneration.cpp | 8 ++++-- .../gc/shenandoah/shenandoahGeneration.hpp | 2 +- .../share/gc/shenandoah/shenandoahOldGC.cpp | 7 +---- .../gc/shenandoah/shenandoahOldGeneration.cpp | 28 +++++++++++++++++++ .../gc/shenandoah/shenandoahOldGeneration.hpp | 2 ++ 5 files changed, 37 insertions(+), 10 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index a0341f76b2d91..911aa4fd82095 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -199,9 +199,11 @@ bool ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) { parallel_heap_region_iterate(&cl); heap->assert_pinned_region_status(); - // Also capture update_watermark for old-gen regions. - ShenandoahCaptureUpdateWaterMarkForOld old_cl(complete_marking_context()); - heap->old_generation()->parallel_heap_region_iterate(&old_cl); + if (generation_mode() == YOUNG) { + // Also capture update_watermark for old-gen regions. + ShenandoahCaptureUpdateWaterMarkForOld old_cl(complete_marking_context()); + heap->old_generation()->parallel_heap_region_iterate(&old_cl); + } } { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index 730ef46bfe9b5..ce02391a7a9d6 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -93,7 +93,7 @@ class ShenandoahGeneration : public CHeapObj { virtual void prepare_gc(bool do_old_gc_bootstrap); // Return true iff prepared collection set includes at least one old-gen HeapRegion. - bool prepare_regions_and_collection_set(bool concurrent); + virtual bool prepare_regions_and_collection_set(bool concurrent); // Cancel marking (used by Full collect and when cancelling cycle). void cancel_marking(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp index a4ce1da13691d..03e7608692980 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp @@ -97,12 +97,7 @@ void ShenandoahOldGC::op_final_mark() { // We need to do this because weak root cleaning reports the number of dead handles JvmtiTagMap::set_needs_cleaning(); - { - ShenandoahGCPhase phase(ShenandoahPhaseTimings::choose_cset); - ShenandoahHeapLocker locker(heap->lock()); - // Old-gen choose_collection_set() does not directly manipulate heap->collection_set() so no need to clear it. - _generation->heuristics()->choose_collection_set(nullptr, nullptr); - } + _generation->prepare_regions_and_collection_set(true); heap->set_unload_classes(false); heap->prepare_concurrent_roots(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp index 99b23e867717c..6bc89ec1bd5fd 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp @@ -26,9 +26,11 @@ #include "gc/shared/strongRootsScope.hpp" #include "gc/shenandoah/shenandoahAsserts.hpp" +#include "gc/shenandoah/shenandoahFreeSet.hpp" #include "gc/shenandoah/shenandoahHeap.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahHeapRegion.hpp" +#include "gc/shenandoah/shenandoahMarkClosures.hpp" #include "gc/shenandoah/shenandoahMark.inline.hpp" #include "gc/shenandoah/shenandoahOldGeneration.hpp" #include "gc/shenandoah/shenandoahOopClosures.inline.hpp" @@ -179,3 +181,29 @@ void ShenandoahOldGeneration::purge_satb_buffers(bool abandon) { bool ShenandoahOldGeneration::contains(oop obj) const { return ShenandoahHeap::heap()->is_in_old(obj); } + +bool ShenandoahOldGeneration::prepare_regions_and_collection_set(bool concurrent) { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + assert(!heap->is_full_gc_in_progress(), "Only for concurrent and degenerated GC"); + + { + ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_update_region_states : ShenandoahPhaseTimings::degen_gc_final_update_region_states); + ShenandoahFinalMarkUpdateRegionStateClosure cl(complete_marking_context()); + + parallel_heap_region_iterate(&cl); + heap->assert_pinned_region_status(); + } + + { + ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::choose_cset : ShenandoahPhaseTimings::degen_gc_choose_cset); + ShenandoahHeapLocker locker(heap->lock()); + heuristics()->choose_collection_set(nullptr, nullptr); + } + + { + ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_rebuild_freeset : ShenandoahPhaseTimings::degen_gc_final_rebuild_freeset); + ShenandoahHeapLocker locker(heap->lock()); + heap->free_set()->rebuild(); + } + return false; +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp index f2f39b79cdfdd..94151d5c374da 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp @@ -46,6 +46,8 @@ class ShenandoahOldGeneration : public ShenandoahGeneration { void set_concurrent_mark_in_progress(bool in_progress) override; + bool prepare_regions_and_collection_set(bool concurrent) override; + // We leave the SATB barrier on for the entirety of the old generation // marking phase. In some cases, this can cause a write to a perfectly // reachable oop to enqueue a pointer that later becomes garbage (because From 9f0f23b719fd90387c4e735ced5b825c4d56ff92 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Fri, 20 Aug 2021 21:17:56 +0000 Subject: [PATCH 058/254] Preserve and restore original element count for card table barrier Reviewed-by: rkennke --- .../shenandoahBarrierSetAssembler_x86.cpp | 34 +++++++++++++++---- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp index 96d9a42a3e40a..f6408ae445d3e 100644 --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp @@ -121,6 +121,30 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0; if (is_reference_type(type)) { + if (ShenandoahHeap::heap()->mode()->is_generational()) { + bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0; + bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0; + bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops); + + // We need to squirrel away the original element count because the + // array copy assembly will destroy the value and we need it for the + // card marking barrier. +#ifdef _LP64 + if (!checkcast) { + if (!obj_int) { + // Save count for barrier + __ movptr(r11, count); + } else if (disjoint) { + // Save dst in r11 in the disjoint case + __ movq(r11, dst); + } + } +#else +if (disjoint) { + __ mov(rdx, dst); // save 'to' + } +#endif + } if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahIUBarrier || ShenandoahLoadRefBarrier) { #ifdef _LP64 @@ -187,16 +211,11 @@ void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, Dec bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0; bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0; bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops); -#ifdef _LP64 - Register tmp = rscratch1; -#else Register tmp = rax; -#endif if (is_reference_type(type)) { #ifdef _LP64 -#if COMPILER2_OR_JVMCI - if (VM_Version::supports_avx512vlbw() && MaxVectorSize >= 32 && !checkcast) { + if (!checkcast) { if (!obj_int) { // Save count for barrier count = r11; @@ -204,8 +223,9 @@ if (is_reference_type(type)) { // Use the saved dst in the disjoint case dst = r11; } + } else { + tmp = rscratch1; } -# endif #else if (disjoint) { __ mov(dst, rdx); // restore 'to' From 4d100619150c5305bce85b578f01bf5068a4f23c Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Fri, 27 Aug 2021 17:46:28 +0000 Subject: [PATCH 059/254] Add generational full gc support Reviewed-by: rkennke --- .../shenandoahAdaptiveHeuristics.cpp | 8 + .../shenandoahAdaptiveOldHeuristics.cpp | 18 +- .../gc/shenandoah/shenandoahBarrierSet.cpp | 3 + .../share/gc/shenandoah/shenandoahFreeSet.cpp | 18 +- .../share/gc/shenandoah/shenandoahFullGC.cpp | 452 +++++++++++++++--- .../gc/shenandoah/shenandoahGeneration.cpp | 6 + .../gc/shenandoah/shenandoahGeneration.hpp | 1 + .../share/gc/shenandoah/shenandoahHeap.cpp | 196 ++++---- .../share/gc/shenandoah/shenandoahHeap.hpp | 3 + .../gc/shenandoah/shenandoahHeapRegion.cpp | 7 +- .../shenandoahHeapRegion.inline.hpp | 1 - .../shenandoah/shenandoahMarkingContext.cpp | 8 +- .../gc/shenandoah/shenandoahOopClosures.hpp | 19 + .../shenandoahOopClosures.inline.hpp | 12 + .../gc/shenandoah/shenandoahPhaseTimings.hpp | 1 + .../shenandoahScanRemembered.inline.hpp | 44 +- .../share/gc/shenandoah/shenandoahUtils.cpp | 1 + .../gc/shenandoah/shenandoahVerifier.cpp | 50 +- .../gc/shenandoah/shenandoahVerifier.hpp | 17 +- .../shenandoah/shenandoahYoungGeneration.cpp | 15 - .../shenandoah/shenandoahYoungGeneration.hpp | 2 - .../gc/shenandoah/shenandoah_globals.hpp | 7 +- 22 files changed, 688 insertions(+), 201 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp index ef8227ee4bf4e..dba883fa74ace 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp @@ -201,6 +201,10 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { size_t available = _generation->available(); size_t allocated = _generation->bytes_allocated_since_gc_start(); + log_debug(gc)("should_start_gc? available: " SIZE_FORMAT ", soft_max_capacity: " SIZE_FORMAT + ", max_capacity: " SIZE_FORMAT ", allocated: " SIZE_FORMAT, + available, capacity, max_capacity, allocated); + // Make sure the code below treats available without the soft tail. size_t soft_tail = max_capacity - capacity; available = (available > soft_tail) ? (available - soft_tail) : 0; @@ -210,6 +214,10 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { _last_trigger = OTHER; size_t min_threshold = capacity / 100 * ShenandoahMinFreeThreshold; + + log_debug(gc)(" available adjusted to: " SIZE_FORMAT ", min_threshold: " SIZE_FORMAT ", ShenandoahMinFreeThreshold: " SIZE_FORMAT, + available, min_threshold, ShenandoahMinFreeThreshold); + if (available < min_threshold) { log_info(gc)("Trigger (%s): Free (" SIZE_FORMAT "%s) is below minimum threshold (" SIZE_FORMAT "%s)", _generation->name(), diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveOldHeuristics.cpp index 66d6e018f0187..7f58bb7b72b05 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveOldHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveOldHeuristics.cpp @@ -201,8 +201,18 @@ bool ShenandoahAdaptiveOldHeuristics::should_start_gc() { size_t max_capacity = _generation->max_capacity(); size_t capacity = _generation->soft_max_capacity(); size_t available = _generation->available(); + // TODO: Fix implementation of old_generation->bytes_allocated_since_gc_start() to represent bytes promoted since + // start of most recent OLD collection. + + // Note further: available is the difference between soft_capacity and in_use. So soft_tail has already been removed + // from this total. It is redundant to remove it again below. + size_t allocated = _generation->bytes_allocated_since_gc_start(); + log_debug(gc)("should_start_old_gc? available: " SIZE_FORMAT ", soft_max_capacity: " SIZE_FORMAT ", max_capacity: " SIZE_FORMAT, + available, capacity, max_capacity); + log_debug(gc)(" allocated: " SIZE_FORMAT, allocated); + // Make sure the code below treats available without the soft tail. size_t soft_tail = max_capacity - capacity; available = (available > soft_tail) ? (available - soft_tail) : 0; @@ -211,7 +221,11 @@ bool ShenandoahAdaptiveOldHeuristics::should_start_gc() { double rate = _allocation_rate.sample(allocated); _last_trigger = OTHER; - size_t min_threshold = capacity / 100 * ShenandoahMinFreeThreshold; + size_t min_threshold = (capacity * ShenandoahMinFreeThreshold) / 100; + + log_debug(gc)(" available adjusted to: " SIZE_FORMAT ", min_threshold: " SIZE_FORMAT ", ShenandoahMinFreeThreshold: " SIZE_FORMAT, + available, min_threshold, ShenandoahMinFreeThreshold); + if (available < min_threshold) { log_info(gc)("Trigger (%s): Free (" SIZE_FORMAT "%s) is below minimum threshold (" SIZE_FORMAT "%s)", _generation->name(), @@ -220,7 +234,7 @@ bool ShenandoahAdaptiveOldHeuristics::should_start_gc() { return true; } - // Check if are need to learn a bit about the application + // Check if we need to learn a bit about the application const size_t max_learn = ShenandoahLearningSteps; if (_gc_times_learned < max_learn) { size_t init_threshold = capacity / 100 * ShenandoahInitFreeThreshold; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp index 8c68b56d7c3f8..dabed4cc238b8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp @@ -122,6 +122,9 @@ void ShenandoahBarrierSet::on_thread_detach(Thread *thread) { ShenandoahHeap* const heap = ShenandoahHeap::heap(); PLAB* plab = ShenandoahThreadLocalData::plab(thread); + // CAUTION: retire_plab may register the remnant filler object with the remembered set scanner without a lock. + // This is safe iff it is assured that each PLAB is a whole-number multiple of card-mark memory size and each + // PLAB is aligned with the start of each card's memory range. heap->retire_plab(plab); // SATB protocol requires to keep alive reacheable oops from roots at the beginning of GC diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index 5c9874e607b3f..745f79f8b64f8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -204,15 +204,21 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah } if (result != NULL) { - // Allocation successful, bump stats: - if (req.is_mutator_alloc()) { - increase_used(size * HeapWordSize); - } // Record actual allocation size req.set_actual_size(size); - if (req.is_gc_alloc()) { + // Allocation successful, bump stats: + if (req.is_mutator_alloc()) { + increase_used(size * HeapWordSize); + } else if (req.is_gc_alloc()) { + // For GC allocations, we advance update_watermark because the objects relocated into this memory during + // evacuation are not updated during evacuation. For both young and old regions r, it is essential that all + // PLABs be made parsable at the end of evacuation. This is enabled by retiring all plabs at end of evacuation. + // TODO: Making a PLAB parsable involves placing a filler object in its remnant memory but does not require + // that the PLAB be disabled for all future purposes. We may want to introduce a new service to make the + // PLABs parsable while still allowing the PLAB to serve future allocation requests that arise during the + // next evacuation pass. r->set_update_watermark(r->top()); } @@ -487,7 +493,7 @@ void ShenandoahFreeSet::rebuild() { assert(!is_mutator_free(idx), "We are about to add it, it shouldn't be there already"); _mutator_free_bitmap.set_bit(idx); - log_debug(gc)(" Setting _mutator_free_bitmap bit for " SIZE_FORMAT, idx); + log_debug(gc)(" Setting Region " SIZE_FORMAT " _mutator_free_bitmap bit to true", idx); } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index 7466b8b6164c4..3a2dc5b3698b6 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -63,6 +63,67 @@ #include "utilities/growableArray.hpp" #include "gc/shared/workgroup.hpp" +// After Full GC is done, reconstruct the remembered set by iterating over OLD regions, +// registering all objects between bottom() and top(), and setting remembered set cards to +// DIRTY if they hold interesting pointers. +class ShenandoahReconstructRememberedSetTask : public AbstractGangTask { +private: + ShenandoahRegionIterator _regions; + +public: + ShenandoahReconstructRememberedSetTask() : + AbstractGangTask("Shenandoah Reset Bitmap") { } + + void work(uint worker_id) { + ShenandoahParallelWorkerSession worker_session(worker_id); + ShenandoahHeapRegion* r = _regions.next(); + ShenandoahHeap* heap = ShenandoahHeap::heap(); + RememberedScanner* scanner = heap->card_scan(); + ShenandoahSetRememberedCardsToDirtyClosure dirty_cards_for_interesting_pointers; + + while (r != NULL) { + if (r->is_old() && r->is_active()) { + HeapWord* obj_addr = r->bottom(); + if (r->is_humongous_start()) { + // First, clear the remembered set + oop obj = cast_to_oop(obj_addr); + size_t size = obj->size(); + HeapWord* end_object = r->bottom() + size; + + // First, clear the remembered set + scanner->reset_remset(r->bottom(), size); + size_t region_index = r->index(); + ShenandoahHeapRegion* humongous_region = heap->get_region(region_index); + do { + scanner->reset_object_range(humongous_region->bottom(), humongous_region->end()); + region_index++; + humongous_region = heap->get_region(region_index); + } while (humongous_region->bottom() < end_object); + + // Then register the humongous object and DIRTY relevant remembered set cards + scanner->register_object_wo_lock(obj_addr); + obj->oop_iterate(&dirty_cards_for_interesting_pointers); + } else if (!r->is_humongous()) { + // First, clear the remembered set + scanner->reset_remset(r->bottom(), ShenandoahHeapRegion::region_size_words()); + scanner->reset_object_range(r->bottom(), r->end()); + + // Then iterate over all objects, registering object and DIRTYing relevant remembered set cards + HeapWord* t = r->top(); + while (obj_addr < t) { + oop obj = cast_to_oop(obj_addr); + size_t size = obj->size(); + scanner->register_object_wo_lock(obj_addr); + obj_addr += obj->oop_iterate_size(&dirty_cards_for_interesting_pointers); + } + } // else, ignore humongous continuation region + } + // else, this region is FREE or YOUNG or inactive and we can ignore it. + r = _regions.next(); + } + } +}; + ShenandoahFullGC::ShenandoahFullGC() : _gc_timer(ShenandoahHeap::heap()->gc_timer()), _preserved_marks(new PreservedMarksSet(true)) {} @@ -96,10 +157,6 @@ void ShenandoahFullGC::entry_full(GCCause::Cause cause) { } void ShenandoahFullGC::op_full(GCCause::Cause cause) { - if (ShenandoahHeap::heap()->mode()->is_generational()) { - fatal("Full GC not yet supported for generational mode."); - } - ShenandoahMetricsSnapshot metrics; metrics.snap_before(); @@ -119,10 +176,9 @@ void ShenandoahFullGC::op_full(GCCause::Cause cause) { void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) { ShenandoahHeap* heap = ShenandoahHeap::heap(); + // Since we may arrive here from degenerated GC failure of either young or old, establish generation as GLOBAL. + heap->set_gc_generation(heap->global_generation()); - if (ShenandoahHeap::heap()->mode()->is_generational()) { - fatal("Full GC not yet supported for generational mode in do_it()."); - } if (ShenandoahVerify) { heap->verifier()->verify_before_fullgc(); } @@ -243,6 +299,12 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) { _preserved_marks->restore(heap->workers()); BiasedLocking::restore_marks(); _preserved_marks->reclaim(); + + if (heap->mode()->is_generational()) { + ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_reconstruct_remembered_set); + ShenandoahReconstructRememberedSetTask task; + heap->workers()->run_task(&task); + } } // Resize metaspace @@ -258,7 +320,11 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) { heap->set_full_gc_in_progress(false); if (ShenandoahVerify) { - heap->verifier()->verify_after_fullgc(); + if (heap->mode()->is_generational()) { + heap->verifier()->verify_after_generational_fullgc(); + } else { + heap->verifier()->verify_after_fullgc(); + } } if (VerifyAfterGC) { @@ -308,6 +374,208 @@ void ShenandoahFullGC::phase1_mark_heap() { heap->parallel_cleaning(true /* full_gc */); } +class ShenandoahPrepareForCompactionTask : public AbstractGangTask { +private: + PreservedMarksSet* const _preserved_marks; + ShenandoahHeap* const _heap; + ShenandoahHeapRegionSet** const _worker_slices; + size_t const _num_workers; + size_t *_old_used; + size_t *_young_used; + +public: + ShenandoahPrepareForCompactionTask(PreservedMarksSet *preserved_marks, ShenandoahHeapRegionSet **worker_slices, + size_t num_workers, size_t* old_used, size_t* young_used); + + static bool is_candidate_region(ShenandoahHeapRegion* r) { + // Empty region: get it into the slice to defragment the slice itself. + // We could have skipped this without violating correctness, but we really + // want to compact all live regions to the start of the heap, which sometimes + // means moving them into the fully empty regions. + if (r->is_empty()) return true; + + // Can move the region, and this is not the humongous region. Humongous + // moves are special cased here, because their moves are handled separately. + return r->is_stw_move_allowed() && !r->is_humongous(); + } + + void work(uint worker_id); + void add_to_young_used(size_t worker_id, size_t amount); + void add_to_old_used(size_t worker_id, size_t amount); + size_t young_used(); + size_t old_used(); +}; + +class ShenandoahPrepareForGenerationalCompactionObjectClosure : public ObjectClosure { +private: + ShenandoahPrepareForCompactionTask* _compactor; + PreservedMarks* const _preserved_marks; + ShenandoahHeap* const _heap; + GrowableArray& _empty_regions; + int _empty_regions_pos; + ShenandoahHeapRegion* _old_to_region; + ShenandoahHeapRegion* _young_to_region; + ShenandoahHeapRegion* _from_region; + ShenandoahRegionAffiliation _from_affiliation; + HeapWord* _old_compact_point; + HeapWord* _young_compact_point; + uint _worker_id; + +public: + ShenandoahPrepareForGenerationalCompactionObjectClosure(ShenandoahPrepareForCompactionTask* compactor, + PreservedMarks* preserved_marks, + GrowableArray& empty_regions, + ShenandoahHeapRegion* old_to_region, + ShenandoahHeapRegion* young_to_region, uint worker_id) : + _compactor(compactor), + _preserved_marks(preserved_marks), + _heap(ShenandoahHeap::heap()), + _empty_regions(empty_regions), + _empty_regions_pos(0), + _old_to_region(old_to_region), + _young_to_region(young_to_region), + _from_region(NULL), + _old_compact_point((old_to_region != nullptr)? old_to_region->bottom(): nullptr), + _young_compact_point((young_to_region != nullptr)? young_to_region->bottom(): nullptr), + _worker_id(worker_id) {} + + void set_from_region(ShenandoahHeapRegion* from_region) { + _from_region = from_region; + _from_affiliation = from_region->affiliation(); + if (_from_region->has_live()) { + if (_from_affiliation == ShenandoahRegionAffiliation::OLD_GENERATION) { + if (_old_to_region == nullptr) { + _old_to_region = from_region; + _old_compact_point = from_region->bottom(); + } + } else { + assert(_from_affiliation == ShenandoahRegionAffiliation::YOUNG_GENERATION, "from_region must be OLD or YOUNG"); + if (_young_to_region == nullptr) { + _young_to_region = from_region; + _young_compact_point = from_region->bottom(); + } + } + } // else, we won't iterate over this _from_region so we don't need to set up to region to hold copies + } + + void finish() { + finish_old_region(); + finish_young_region(); + } + + void finish_old_region() { + if (_old_to_region != nullptr) { + log_debug(gc)("Planned compaction into Old Region " SIZE_FORMAT ", used: " SIZE_FORMAT " tabulated by worker %u", + _old_to_region->index(), _old_compact_point - _old_to_region->bottom(), _worker_id); + _compactor->add_to_old_used(_worker_id, _old_compact_point - _old_to_region->bottom()); + _old_to_region->set_new_top(_old_compact_point); + _old_to_region = nullptr; + } + } + + void finish_young_region() { + if (_young_to_region != nullptr) { + log_debug(gc)("Worker %u planned compaction into Young Region " SIZE_FORMAT ", used: " SIZE_FORMAT, + _worker_id, _young_to_region->index(), _young_compact_point - _young_to_region->bottom()); + _compactor->add_to_young_used(_worker_id, _young_compact_point - _young_to_region->bottom()); + _young_to_region->set_new_top(_young_compact_point); + _young_to_region = nullptr; + } + } + + bool is_compact_same_region() { + return (_from_region == _old_to_region) || (_from_region == _young_to_region); + } + + int empty_regions_pos() { + return _empty_regions_pos; + } + + void do_object(oop p) { + assert(_from_region != NULL, "must set before work"); + assert((_from_region->bottom() <= cast_from_oop(p)) && (cast_from_oop(p) < _from_region->top()), + "Object must reside in _from_region"); + assert(_heap->complete_marking_context()->is_marked(p), "must be marked"); + assert(!_heap->complete_marking_context()->allocated_after_mark_start(p), "must be truly marked"); + + size_t obj_size = p->size(); + if (_from_affiliation == ShenandoahRegionAffiliation::OLD_GENERATION) { + assert(_old_to_region != nullptr, "_old_to_region should not be NULL when compacting OLD _from_region"); + if (_old_compact_point + obj_size > _old_to_region->end()) { + ShenandoahHeapRegion* new_to_region; + + log_debug(gc)("Worker %u finishing old region " SIZE_FORMAT ", compact_point: " PTR_FORMAT ", obj_size: " SIZE_FORMAT + ", &compact_point[obj_size]: " PTR_FORMAT ", region end: " PTR_FORMAT, _worker_id, _old_to_region->index(), + p2i(_old_compact_point), obj_size, p2i(_old_compact_point + obj_size), p2i(_old_to_region->end())); + + // Object does not fit. Get a new _old_to_region. + finish_old_region(); + if (_empty_regions_pos < _empty_regions.length()) { + new_to_region = _empty_regions.at(_empty_regions_pos); + _empty_regions_pos++; + new_to_region->set_affiliation(OLD_GENERATION); + } else { + // If we've exhausted the previously selected _old_to_region, we know that the _old_to_region is distinct + // from _from_region. That's because there is always room for _from_region to be compacted into itself. + // Since we're out of empty regions, let's use _from_region to hold the results of its own compaction. + new_to_region = _from_region; + } + + assert(new_to_region != _old_to_region, "must not reuse same OLD to-region"); + assert(new_to_region != NULL, "must not be NULL"); + _old_to_region = new_to_region; + _old_compact_point = _old_to_region->bottom(); + } + + // Object fits into current region, record new location: + assert(_old_compact_point + obj_size <= _old_to_region->end(), "must fit"); + shenandoah_assert_not_forwarded(NULL, p); + _preserved_marks->push_if_necessary(p, p->mark()); + p->forward_to(cast_to_oop(_old_compact_point)); + _old_compact_point += obj_size; + } else { + assert(_from_affiliation == ShenandoahRegionAffiliation::YOUNG_GENERATION, + "_from_region must be OLD_GENERATION or YOUNG_GENERATION"); + + assert(_young_to_region != nullptr, "_young_to_region should not be NULL when compacting YOUNG _from_region"); + if (_young_compact_point + obj_size > _young_to_region->end()) { + ShenandoahHeapRegion* new_to_region; + + + log_debug(gc)("Worker %u finishing young region " SIZE_FORMAT ", compact_point: " PTR_FORMAT ", obj_size: " SIZE_FORMAT + ", &compact_point[obj_size]: " PTR_FORMAT ", region end: " PTR_FORMAT, _worker_id, _young_to_region->index(), + p2i(_young_compact_point), obj_size, p2i(_young_compact_point + obj_size), p2i(_young_to_region->end())); + + // Object does not fit. Get a new _young_to_region. + finish_young_region(); + if (_empty_regions_pos < _empty_regions.length()) { + new_to_region = _empty_regions.at(_empty_regions_pos); + _empty_regions_pos++; + new_to_region->set_affiliation(YOUNG_GENERATION); + } else { + // If we've exhausted the previously selected _young_to_region, we know that the _young_to_region is distinct + // from _from_region. That's because there is always room for _from_region to be compacted into itself. + // Since we're out of empty regions, let's use _from_region to hold the results of its own compaction. + new_to_region = _from_region; + } + + assert(new_to_region != _young_to_region, "must not reuse same OLD to-region"); + assert(new_to_region != NULL, "must not be NULL"); + _young_to_region = new_to_region; + _young_compact_point = _young_to_region->bottom(); + } + + // Object fits into current region, record new location: + assert(_young_compact_point + obj_size <= _young_to_region->end(), "must fit"); + shenandoah_assert_not_forwarded(NULL, p); + _preserved_marks->push_if_necessary(p, p->mark()); + p->forward_to(cast_to_oop(_young_compact_point)); + _young_compact_point += obj_size; + } + } +}; + + class ShenandoahPrepareForCompactionObjectClosure : public ObjectClosure { private: PreservedMarks* const _preserved_marks; @@ -336,14 +604,7 @@ class ShenandoahPrepareForCompactionObjectClosure : public ObjectClosure { void finish_region() { assert(_to_region != NULL, "should not happen"); - if (_heap->mode()->is_generational() && _to_region->affiliation() == FREE) { - // TODO: Changing this region to young during compaction may not be - // technically correct here because it completely disregards the ages - // and origins of the objects being moved. It is, however, certainly - // more correct than putting live objects into a region without a - // generational affiliation. - _to_region->set_affiliation(YOUNG_GENERATION); - } + assert(!_heap->mode()->is_generational(), "Generational GC should use different Closure"); _to_region->set_new_top(_compact_point); } @@ -389,52 +650,69 @@ class ShenandoahPrepareForCompactionObjectClosure : public ObjectClosure { } }; -class ShenandoahPrepareForCompactionTask : public AbstractGangTask { -private: - PreservedMarksSet* const _preserved_marks; - ShenandoahHeap* const _heap; - ShenandoahHeapRegionSet** const _worker_slices; -public: - ShenandoahPrepareForCompactionTask(PreservedMarksSet *preserved_marks, ShenandoahHeapRegionSet **worker_slices) : +ShenandoahPrepareForCompactionTask::ShenandoahPrepareForCompactionTask(PreservedMarksSet *preserved_marks, + ShenandoahHeapRegionSet **worker_slices, + size_t num_workers, size_t* old_used, size_t* young_used) : AbstractGangTask("Shenandoah Prepare For Compaction"), - _preserved_marks(preserved_marks), - _heap(ShenandoahHeap::heap()), _worker_slices(worker_slices) { + _preserved_marks(preserved_marks), _heap(ShenandoahHeap::heap()), + _worker_slices(worker_slices), _num_workers(num_workers), + _old_used(old_used), _young_used(young_used) { + for (size_t i = 0; i < _num_workers; i++) { + _old_used[i] = 0; + _young_used[i] = 0; } +} - static bool is_candidate_region(ShenandoahHeapRegion* r) { - // Empty region: get it into the slice to defragment the slice itself. - // We could have skipped this without violating correctness, but we really - // want to compact all live regions to the start of the heap, which sometimes - // means moving them into the fully empty regions. - if (r->is_empty()) return true; - // Can move the region, and this is not the humongous region. Humongous - // moves are special cased here, because their moves are handled separately. - return r->is_stw_move_allowed() && !r->is_humongous(); +void ShenandoahPrepareForCompactionTask::work(uint worker_id) { + ShenandoahParallelWorkerSession worker_session(worker_id); + ShenandoahHeapRegionSet* slice = _worker_slices[worker_id]; + ShenandoahHeapRegionSetIterator it(slice); + ShenandoahHeapRegion* from_region = it.next(); + // No work? + if (from_region == NULL) { + return; } - void work(uint worker_id) { - ShenandoahParallelWorkerSession worker_session(worker_id); - ShenandoahHeapRegionSet* slice = _worker_slices[worker_id]; - ShenandoahHeapRegionSetIterator it(slice); - ShenandoahHeapRegion* from_region = it.next(); - // No work? - if (from_region == NULL) { - return; - } + // Sliding compaction. Walk all regions in the slice, and compact them. + // Remember empty regions and reuse them as needed. + ResourceMark rm; - // Sliding compaction. Walk all regions in the slice, and compact them. - // Remember empty regions and reuse them as needed. - ResourceMark rm; + GrowableArray empty_regions((int)_heap->num_regions()); + + if (_heap->mode()->is_generational()) { + ShenandoahHeapRegion* old_to_region = (from_region->is_old())? from_region: nullptr; + ShenandoahHeapRegion* young_to_region = (from_region->is_young())? from_region: nullptr; + ShenandoahPrepareForGenerationalCompactionObjectClosure cl(this, _preserved_marks->get(worker_id), empty_regions, + old_to_region, young_to_region, worker_id); + while (from_region != NULL) { + assert(is_candidate_region(from_region), "Sanity"); + log_debug(gc)("Worker %u compacting %s Region " SIZE_FORMAT " which had used " SIZE_FORMAT " and %s live", + worker_id, affiliation_name(from_region->affiliation()), + from_region->index(), from_region->used(), from_region->has_live()? "has": "does not have"); + cl.set_from_region(from_region); + if (from_region->has_live()) { + _heap->marked_object_iterate(from_region, &cl); + } - GrowableArray empty_regions((int)_heap->num_regions()); + // Compacted the region to somewhere else? From-region is empty then. + if (!cl.is_compact_same_region()) { + empty_regions.append(from_region); + } + from_region = it.next(); + } + cl.finish(); + // Mark all remaining regions as empty + for (int pos = cl.empty_regions_pos(); pos < empty_regions.length(); ++pos) { + ShenandoahHeapRegion* r = empty_regions.at(pos); + r->set_new_top(r->bottom()); + } + } else { ShenandoahPrepareForCompactionObjectClosure cl(_preserved_marks->get(worker_id), empty_regions, from_region); - while (from_region != NULL) { assert(is_candidate_region(from_region), "Sanity"); - cl.set_from_region(from_region); if (from_region->has_live()) { _heap->marked_object_iterate(from_region, &cl); @@ -454,7 +732,45 @@ class ShenandoahPrepareForCompactionTask : public AbstractGangTask { r->set_new_top(r->bottom()); } } -}; +} + +// Accumulate HeapWords of memory used in young-gen memory. +void ShenandoahPrepareForCompactionTask::add_to_young_used(size_t worker_id, size_t amount) { + log_debug(gc)("Adding to _young_used for worker_id: " SIZE_FORMAT ", amount: " SIZE_FORMAT, worker_id, amount); + _young_used[worker_id] += amount; +} + +// Accumulate HeapWords of memory used in old-gen memory. +void ShenandoahPrepareForCompactionTask::add_to_old_used(size_t worker_id, size_t amount) { + log_debug(gc)("Adding to _old_used for worker_id: " SIZE_FORMAT ", amount: " SIZE_FORMAT, worker_id, amount); + _old_used[worker_id] += amount; +} + +// Return total number of bytes used in young-gen memory +size_t ShenandoahPrepareForCompactionTask::young_used() { + size_t result = 0; + log_debug(gc)("Calculating young_used by accumulating worker totals"); + for (size_t i = 0; i < _num_workers; i++) { + log_debug(gc)(" worker [" SIZE_FORMAT "] contributed " SIZE_FORMAT, i, _young_used[i]); + result += _young_used[i]; + } + result *= HeapWordSize; + log_debug(gc)("Accumulated _young_used is: " SIZE_FORMAT, result); + return result; +} + +// Return total number of bytes used in old-gen memory +size_t ShenandoahPrepareForCompactionTask::old_used() { + size_t result = 0; + log_debug(gc)("Calculating old_used by accumulating worker totals"); + for (size_t i = 0; i < _num_workers; i++) { + log_debug(gc)(" worker [" SIZE_FORMAT "] contributed " SIZE_FORMAT, i, _old_used[i]); + result += _old_used[i]; + } + log_debug(gc)("Accumulated _old_used is: " SIZE_FORMAT, result); + result *= HeapWordSize; + return result; +} void ShenandoahFullGC::calculate_target_humongous_objects() { ShenandoahHeap* heap = ShenandoahHeap::heap(); @@ -476,6 +792,17 @@ void ShenandoahFullGC::calculate_target_humongous_objects() { log_debug(gc)("Full GC calculating target humongous objects from end " SIZE_FORMAT, to_end); for (size_t c = heap->num_regions(); c > 0; c--) { ShenandoahHeapRegion *r = heap->get_region(c - 1); + + if (r->is_humongous_start() && heap->mode()->is_generational()) { + oop obj = cast_to_oop(r->bottom()); + size_t humongous_bytes = obj->size() * HeapWordSize; + log_debug(gc)("Adjusting used for humongous %s object by " SIZE_FORMAT, r->is_old()? "OLD": "YOUNG", humongous_bytes); + if (r->is_old()) { + heap->old_generation()->increase_used(humongous_bytes); + } else { + heap->young_generation()->increase_used(humongous_bytes); + } + } if (r->is_humongous_continuation() || (r->new_top() == r->bottom())) { // To-region candidate: record this, and continue scan to_begin = r->index(); @@ -732,14 +1059,29 @@ void ShenandoahFullGC::phase2_calculate_target_addresses(ShenandoahHeapRegionSet heap->heap_region_iterate(&ecl); } + if (heap->mode()->is_generational()) { + heap->young_generation()->clear_used(); + heap->old_generation()->clear_used(); + } + // Compute the new addresses for regular objects { ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_calculate_addresses_regular); distribute_slices(worker_slices); - ShenandoahPrepareForCompactionTask task(_preserved_marks, worker_slices); + size_t num_workers = heap->workers()->total_workers(); + size_t old_used[num_workers]; + size_t young_used[num_workers]; + ShenandoahPrepareForCompactionTask task(_preserved_marks, worker_slices, num_workers, old_used, young_used); heap->workers()->run_task(&task); + + if (heap->mode()->is_generational()) { + log_debug(gc)("Usage after compacting regular objects is young: " SIZE_FORMAT ", old: " SIZE_FORMAT, + task.young_used(), task.old_used()); + heap->young_generation()->increase_used(task.young_used()); + heap->old_generation()->increase_used(task.old_used()); + } } // Compute the new addresses for humongous objects @@ -1090,13 +1432,13 @@ void ShenandoahFullGC::phase4_compact_objects(ShenandoahHeapRegionSet** worker_s ShenandoahPostCompactClosure post_compact; heap->heap_region_iterate(&post_compact); heap->set_used(post_compact.get_live()); + if (heap->mode()->is_generational()) { + log_info(gc)("FullGC done: GLOBAL usage: " SIZE_FORMAT ", young usage: " SIZE_FORMAT ", old usage: " SIZE_FORMAT, + post_compact.get_live(), heap->young_generation()->used(), heap->old_generation()->used()); + } heap->collection_set()->clear(); heap->free_set()->rebuild(); - - if (heap->mode()->is_generational()) { - heap->young_generation()->promote_all_regions(); - } } heap->clear_cancelled_gc(true /* clear oom handler */); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index 911aa4fd82095..08a61ba22d61f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -322,6 +322,12 @@ void ShenandoahGeneration::decrement_affiliated_region_count() { _affiliated_region_count--; } +void ShenandoahGeneration::clear_used() { + assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at a safepoint"); + // Do this atomically to assure visibility to other threads, even though these other threads may be idle "right now".. + Atomic::store(&_used, (size_t)0); +} + void ShenandoahGeneration::increase_used(size_t bytes) { Atomic::add(&_used, bytes); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index ce02391a7a9d6..367973bbfdf86 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -134,6 +134,7 @@ class ShenandoahGeneration : public CHeapObj { void increment_affiliated_region_count(); void decrement_affiliated_region_count(); + void clear_used(); void increase_used(size_t bytes); void decrease_used(size_t bytes); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 22a1e22b82b57..19aade27fa864 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -854,6 +854,7 @@ HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) return gclab->allocate(size); } +// Establish a new PLAB and allocate size HeapWords within it. HeapWord* ShenandoahHeap::allocate_from_plab_slow(Thread* thread, size_t size) { // New object should fit the PLAB size size_t min_size = MAX2(size, PLAB::min_size()); @@ -876,6 +877,9 @@ HeapWord* ShenandoahHeap::allocate_from_plab_slow(Thread* thread, size_t size) { // Retire current PLAB, and allocate a new one. PLAB* plab = ShenandoahThreadLocalData::plab(thread); + // CAUTION: retire_plab may register the remnant filler object with the remembered set scanner without a lock. This + // is safe iff it is assured that each PLAB is a whole-number multiple of card-mark memory size and each PLAB is + // aligned with the start of a card's memory range. retire_plab(plab); size_t actual_size = 0; @@ -911,6 +915,8 @@ void ShenandoahHeap::retire_plab(PLAB* plab) { // If retiring the plab created a filler object, then we // need to register it with our card scanner so it can // safely walk the region backing the plab. + log_debug(gc)("retire_plab() is registering remnant of size " SIZE_FORMAT " at " PTR_FORMAT, + plab->waste() - waste, p2i(top)); card_scan()->register_object_wo_lock(top); } } @@ -1247,6 +1253,17 @@ class ShenandoahRetireGCLABClosure : public ThreadClosure { PLAB* plab = ShenandoahThreadLocalData::plab(thread); assert(plab != NULL, "PLAB should be initialized for %s", thread->name()); + // TODO; Retiring a PLAB disables it so it cannot support future allocations. This is overkill. For old-gen + // regions, the important thing is to make the memory parsable by the remembered-set scanning code that drives + // the update-refs processing that follows. After the updating of old-gen references is done, it is ok to carve + // this remnant object into smaller pieces during the subsequent evacuation pass, as long as the PLAB is made parsable + // again before the next update-refs phase. + if (plab->top() != nullptr) { + ShenandoahHeapRegion* r = ShenandoahHeap::heap()->heap_region_containing(plab->top()); + log_debug(gc)("Retiring plab with top: " PTR_FORMAT ", hard_end: " PTR_FORMAT + " + AlignmentReserve, in %s Region " SIZE_FORMAT, + p2i(plab->top()), p2i(plab->top() + plab->words_remaining()), affiliation_name(r->affiliation()), r->index()); + } // else, don't bother to report retirement ShenandoahHeap::heap()->retire_plab(plab); if (_resize && ShenandoahThreadLocalData::plab_size(thread) > 0) { ShenandoahThreadLocalData::set_plab_size(thread, 0); @@ -2591,6 +2608,8 @@ void ShenandoahHeap::verify_rem_set_at_mark() { ShenandoahVerifyRemSetClosure check_interesting_pointers(true); ShenandoahMarkingContext* ctx; + log_debug(gc)("Verifying remembered set at %s mark", doing_mixed_evacuations()? "mixed": "young"); + if (doing_mixed_evacuations()) { ctx = mark_context; } else { @@ -2623,7 +2642,7 @@ void ShenandoahHeap::verify_rem_set_at_mark() { HeapWord* t = r->top(); while (obj_addr < t) { oop obj = oop(obj_addr); - // ctx->is_marked() returns true if mark bit set or if obj above TAMS. + // ctx->is_marked() returns true if mark bit set (TAMS not relevant here) if (!ctx || ctx->is_marked(obj)) { // For regular objects (not object arrays), if the card holding the start of the object is dirty, // we do not need to verify that cards spanning interesting pointers within this object are dirty. @@ -2636,15 +2655,13 @@ void ShenandoahHeap::verify_rem_set_at_mark() { "Verify init-mark remembered set violation", "object not properly registered", __FILE__, __LINE__); } obj_addr += obj->size(); + } // Else, this object is not live so we don't verify dirty cards contained therein. + + if (ctx) { + // TAMS not relevant here + obj_addr = ctx->get_next_marked_addr(obj_addr, t); } else { - // This object is not live so we don't verify dirty cards contained therein - ShenandoahHeapRegion* r = heap_region_containing(obj_addr); - HeapWord* tams = ctx->top_at_mark_start(r); - if (obj_addr >= tams) { - obj_addr += obj->size(); - } else { - obj_addr = ctx->get_next_marked_addr(obj_addr, tams); - } + obj_addr += obj->size(); } } } // else, we ignore humongous continuation region @@ -2652,6 +2669,80 @@ void ShenandoahHeap::verify_rem_set_at_mark() { } // all regions have been processed } +void ShenandoahHeap::help_verify_region_rem_set(ShenandoahHeapRegion* r, ShenandoahMarkingContext* ctx, HeapWord* from, + HeapWord* top, HeapWord* registration_watermark, const char* message) { + RememberedScanner* scanner = card_scan(); + ShenandoahVerifyRemSetClosure check_interesting_pointers(false); + + HeapWord* obj_addr = from; + if (r->is_humongous_start()) { + oop obj = cast_to_oop(obj_addr); + if (!ctx || ctx->is_marked(obj)) { + size_t card_index = scanner->card_index_for_addr(obj_addr); + // For humongous objects, the typical object is an array, so the following checks may be overkill + // For regular objects (not object arrays), if the card holding the start of the object is dirty, + // we do not need to verify that cards spanning interesting pointers within this object are dirty. + if (!scanner->is_write_card_dirty(card_index) || obj->is_objArray()) { + obj->oop_iterate(&check_interesting_pointers); + } + // else, object's start is marked dirty and obj is not an objArray, so any interesting pointers are covered + } + // else, this humongous object is not live so no need to verify its internal pointers + + if ((obj_addr < registration_watermark) && !scanner->verify_registration(obj_addr, obj->size())) { + ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, obj_addr, NULL, message, + "object not properly registered", __FILE__, __LINE__); + } + } else if (!r->is_humongous()) { + while (obj_addr < top) { + oop obj = cast_to_oop(obj_addr); + // ctx->is_marked() returns true if mark bit set or if obj above TAMS. + if (!ctx || ctx->is_marked(obj)) { + size_t card_index = scanner->card_index_for_addr(obj_addr); + // For regular objects (not object arrays), if the card holding the start of the object is dirty, + // we do not need to verify that cards spanning interesting pointers within this object are dirty. + if (!scanner->is_write_card_dirty(card_index) || obj->is_objArray()) { + obj->oop_iterate(&check_interesting_pointers); + } + // else, object's start is marked dirty and obj is not an objArray, so any interesting pointers are covered + + if ((obj_addr < registration_watermark) && !scanner->verify_registration(obj_addr, obj->size())) { + ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, obj_addr, NULL, message, + "object not properly registered", __FILE__, __LINE__); + } + } // Else, this object is not live so we don't verify dirty cards contained therein. + + if (ctx) { + ShenandoahHeapRegion* r = heap_region_containing(obj_addr); + HeapWord* tams = ctx->top_at_mark_start(r); + if (obj_addr >= tams) { + obj_addr += obj->size(); + } else { + obj_addr = ctx->get_next_marked_addr(obj_addr, tams); + } + } else { + obj_addr += obj->size(); + } + } + } +} + +void ShenandoahHeap::verify_rem_set_after_full_gc() { + shenandoah_assert_safepoint(); + assert(mode()->is_generational(), "Only verify remembered set for generational operational modes"); + + ShenandoahRegionIterator iterator; + + while (iterator.has_next()) { + ShenandoahHeapRegion* r = iterator.next(); + if (r == nullptr) + break; + if (r->is_old() && !r->is_cset()) { + help_verify_region_rem_set(r, nullptr, r->bottom(), r->top(), r->top(), "Remembered set violation at end of Full GC"); + } + } +} + // Assure that the remember set has a dirty card everywhere there is an interesting pointer. Even though // the update-references scan of remembered set only examines cards up to update_watermark, the remembered // set should be valid through top. This examines the write_card_table between bottom() and top() because @@ -2661,13 +2752,10 @@ void ShenandoahHeap::verify_rem_set_at_update_ref() { assert(mode()->is_generational(), "Only verify remembered set for generational operational modes"); ShenandoahRegionIterator iterator; - ShenandoahMarkingContext* mark_context = marking_context(); - RememberedScanner* scanner = card_scan(); - ShenandoahVerifyRemSetClosure check_interesting_pointers(false); ShenandoahMarkingContext* ctx; if (doing_mixed_evacuations()) { - ctx = mark_context; + ctx = marking_context(); } else { ctx = nullptr; } @@ -2677,84 +2765,8 @@ void ShenandoahHeap::verify_rem_set_at_update_ref() { if (r == nullptr) break; if (r->is_old() && !r->is_cset()) { - HeapWord* obj_addr = r->bottom(); - if (r->is_humongous_start()) { - oop obj = oop(obj_addr); - if (!ctx || ctx->is_marked(obj)) { - size_t card_index = scanner->card_index_for_addr(obj_addr); - // For humongous objects, the typical object is an array, so the following checks may be overkill - // For regular objects (not object arrays), if the card holding the start of the object is dirty, - // we do not need to verify that cards spanning interesting pointers within this object are dirty. - if (!scanner->is_write_card_dirty(card_index) || obj->is_objArray()) { - obj->oop_iterate(&check_interesting_pointers); - } - // else, object's start is marked dirty and obj is not an objArray, so any interesting pointers are covered - } - // else, this humongous object is not live so no need to verify its internal pointers - if (!scanner->verify_registration(obj_addr, obj->size())) { - ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, obj_addr, NULL, - "Verify init-update-references remembered set violation", "object not properly registered", __FILE__, __LINE__); - } - } else if (!r->is_humongous()) { - HeapWord* t = r->get_update_watermark(); - while (obj_addr < t) { - oop obj = oop(obj_addr); - // ctx->is_marked() returns true if mark bit set or if obj above TAMS. - if (!ctx || ctx->is_marked(obj)) { - size_t card_index = scanner->card_index_for_addr(obj_addr); - // For regular objects (not object arrays), if the card holding the start of the object is dirty, - // we do not need to verify that cards spanning interesting pointers within this object are dirty. - if (!scanner->is_write_card_dirty(card_index) || obj->is_objArray()) { - obj->oop_iterate(&check_interesting_pointers); - } - // else, object's start is marked dirty and obj is not an objArray, so any interesting pointers are covered - if (!scanner->verify_registration(obj_addr, obj->size())) { - ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, obj_addr, NULL, - "Verify init-update-references remembered set violation", "object not properly registered", __FILE__, __LINE__); - } - obj_addr += obj->size(); - } else { - // This object is not live so we don't verify dirty cards contained therein - ShenandoahHeapRegion* r = heap_region_containing(obj_addr); - HeapWord* tams = ctx->top_at_mark_start(r); - if (obj_addr >= tams) { - obj_addr += obj->size(); - } else { - obj_addr = ctx->get_next_marked_addr(obj_addr, tams); - } - } - } - // Update references only cares about remembered set below update_watermark, but entire remset should be valid - // We're at safepoint and all LABs have been flushed, so we can parse all the way to top(). - t = r->top(); - while (obj_addr < t) { - oop obj = oop(obj_addr); - // ctx->is_marked() returns true if mark bit set or if obj above TAMS. - if (!ctx || ctx->is_marked(obj)) { - size_t card_index = scanner->card_index_for_addr(obj_addr); - // For regular objects (not object arrays), if the card holding the start of the object is dirty, - // we do not need to verify that cards spanning interesting pointers within this object are dirty. - if (!scanner->is_write_card_dirty(card_index) || obj->is_objArray()) { - obj->oop_iterate(&check_interesting_pointers); - } - // else, object's start is marked dirty and obj is not an objArray, so any interesting pointers are covered - if (!scanner->verify_registration(obj_addr, obj->size())) { - ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, obj_addr, NULL, - "Verify init-update-references remembered set violation", "object not properly registered", __FILE__, __LINE__); - } - obj_addr += obj->size(); - } else { - // This object is not live so we don't verify dirty cards contained therein - ShenandoahHeapRegion* r = heap_region_containing(obj_addr); - HeapWord* tams = ctx->top_at_mark_start(r); - if (obj_addr >= tams) { - obj_addr += obj->size(); - } else { - obj_addr = ctx->get_next_marked_addr(obj_addr, tams); - } - } - } - } - } // else, we don't care about this region + help_verify_region_rem_set(r, ctx, r->bottom(), r->top(), r->get_update_watermark(), + "Remembered set violation at init-update-references"); + } } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index e25a0da2cd064..8fd4934499167 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -205,6 +205,7 @@ class ShenandoahHeap : public CollectedHeap { void verify(VerifyOption vo); void verify_rem_set_at_mark(); void verify_rem_set_at_update_ref(); + void verify_rem_set_after_full_gc(); // ---------- Heap counters and metrics // @@ -219,6 +220,8 @@ class ShenandoahHeap : public CollectedHeap { shenandoah_padding(1); static size_t young_generation_capacity(size_t total_capacity); + void help_verify_region_rem_set(ShenandoahHeapRegion* r, ShenandoahMarkingContext* ctx, + HeapWord* from, HeapWord* top, HeapWord* update_watermark, const char* message); public: void increase_used(size_t bytes); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index d9e2700edbc45..a7486cb98f007 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -559,11 +559,7 @@ void ShenandoahHeapRegion::fill_dead_and_register_for_promotion() { obj_addr += obj->size(); } - // In case top() does not align with a card boundary, it's necessary to fill remainder of memory beyond top(). - if (top() < end()) { - ShenandoahHeap::fill_with_object(top(), end() - top());; - rem_set_scanner->register_object_wo_lock(obj_addr); - } + // Remembered set scanning stops at top() so no need to fill beyond it. } void ShenandoahHeapRegion::oop_iterate_humongous(OopIterateClosure* blk) { @@ -858,6 +854,7 @@ void ShenandoahHeapRegion::set_affiliation(ShenandoahRegionAffiliation new_affil #ifdef ASSERT { + // During full gc, heap->complete_marking_context() is not valid, may equal nullptr. ShenandoahMarkingContext* const ctx = heap->complete_marking_context(); size_t idx = this->index(); HeapWord* top_bitmap = ctx->top_bitmap(this); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp index 12e129ab14df9..5735c7af966f3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp @@ -94,7 +94,6 @@ inline void ShenandoahHeapRegion::internal_increase_live_data(size_t s) { } inline void ShenandoahHeapRegion::clear_live_data() { - log_debug(gc)("SHR::clear_live_data on %s Region " SIZE_FORMAT, affiliation_name(affiliation()), index()); Atomic::store(&_live_data, (size_t)0); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.cpp index 143345c2b236b..f50ad37a4e9e9 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.cpp @@ -70,8 +70,8 @@ void ShenandoahMarkingContext::initialize_top_at_mark_start(ShenandoahHeapRegion // Arrange that the first time we use this bitmap, we clean from bottom to end. _top_bitmaps[idx] = r->end(); - log_debug(gc)("SMC:initialize_top_at_mark_start for region [" PTR_FORMAT ", " PTR_FORMAT "], top_bitmaps set to " PTR_FORMAT, - p2i(r->bottom()), p2i(r->end()), p2i(r->end())); + log_debug(gc)("SMC:initialize_top_at_mark_start for Region " SIZE_FORMAT ", TAMS: " PTR_FORMAT ", TopOfBitMap: " PTR_FORMAT, + r->index(), p2i(bottom), p2i(r->end())); } HeapWord* ShenandoahMarkingContext::top_bitmap(ShenandoahHeapRegion* r) { @@ -82,8 +82,8 @@ void ShenandoahMarkingContext::clear_bitmap(ShenandoahHeapRegion* r) { HeapWord* bottom = r->bottom(); HeapWord* top_bitmap = _top_bitmaps[r->index()]; - log_debug(gc)("SMC:clear_bitmap for %s region [" PTR_FORMAT ", " PTR_FORMAT "], top_bitmap: " PTR_FORMAT, - affiliation_name(r->affiliation()), p2i(r->bottom()), p2i(r->end()), p2i(top_bitmap)); + log_debug(gc)("SMC:clear_bitmap for %s Region " SIZE_FORMAT ", top_bitmap: " PTR_FORMAT, + affiliation_name(r->affiliation()), r->index(), p2i(top_bitmap)); if (r->affiliation() != FREE) { if (top_bitmap > bottom) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp index c24663922d844..df81853dad33a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp @@ -249,4 +249,23 @@ class ShenandoahVerifyRemSetClosure : public BasicOopIterateClosure { virtual void do_oop(oop* p) { work(p); } }; +class ShenandoahSetRememberedCardsToDirtyClosure : public BasicOopIterateClosure { + +protected: + ShenandoahHeap* _heap; + RememberedScanner* _scanner; + +public: + + ShenandoahSetRememberedCardsToDirtyClosure() : + _heap(ShenandoahHeap::heap()), + _scanner(_heap->card_scan()) { } + + template + inline void work(T* p); + + virtual void do_oop(narrowOop* p) { work(p); } + virtual void do_oop(oop* p) { work(p); } +}; + #endif // SHARE_GC_SHENANDOAH_SHENANDOAHOOPCLOSURES_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp index 916b6c230ce73..b518fa2882a2b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp @@ -70,4 +70,16 @@ inline void ShenandoahVerifyRemSetClosure::work(T* p) { } } +template +inline void ShenandoahSetRememberedCardsToDirtyClosure::work(T* p) { + T o = RawAccess<>::oop_load(p); + if (!CompressedOops::is_null(o)) { + oop obj = CompressedOops::decode_not_null(o); + if (_heap->is_in_young(obj)) { + // Found interesting pointer. Mark the containing card as dirty. + _scanner->mark_card_as_dirty((HeapWord*) p); + } + } +} + #endif // SHARE_GC_SHENANDOAH_SHENANDOAHOOPCLOSURES_INLINE_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp index 217261e723a24..76c5df9ae7336 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp @@ -177,6 +177,7 @@ class outputStream; f(full_gc_copy_objects_humong, " Humongous Objects") \ f(full_gc_copy_objects_reset_complete, " Reset Complete Bitmap") \ f(full_gc_copy_objects_rebuild, " Rebuild Region Sets") \ + f(full_gc_reconstruct_remembered_set, " Reconstruct Remembered Set") \ f(full_gc_heapdump_post, " Post Heap Dump") \ \ f(conc_uncommit, "Concurrent Uncommit") \ diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp index 93bda125fc95d..b2a3ac58f9c61 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp @@ -492,7 +492,7 @@ ShenandoahScanRemembered::verify_registration(HeapWord* address, if (!ctx || ctx->is_marked(obj)) { offset += obj->size(); } else { - // This object is not live so don't trust its size() + // If this object is not live, don't trust its size(); all objects above tams are live. ShenandoahHeapRegion* r = heap->heap_region_containing(base_addr + offset); HeapWord* tams = ctx->top_at_mark_start(r); if (base_addr + offset >= tams) { @@ -506,19 +506,26 @@ ShenandoahScanRemembered::verify_registration(HeapWord* address, return false; } + // At this point, offset represents object whose registration we are verifying. We know that at least this object resides + // within this card's memory. + + // Make sure that last_offset is properly set for the enclosing card, but we can't verify this for + // candidate collection-set regions during mixed evacuations, so disable this check in general + // during mixed evacuations. + + ShenandoahHeapRegion* r = heap->heap_region_containing(base_addr + offset); + size_t max_offset = r->top() - base_addr; + if (max_offset > CardTable::card_size_in_words) { + max_offset = CardTable::card_size_in_words; + } + size_t prev_offset; if (!ctx) { - // Make sure that last_offset is properly set for the enclosing card, but we can't verify this for - // candidate collection-set regions during mixed evacuations, so disable this check in general - // during mixed evacuations. - // - // TODO: could do some additional checking during mixed evacuations if we wanted to work harder. - size_t prev_offset = offset; do { HeapWord* obj_addr = base_addr + offset; oop obj = oop(base_addr + offset); prev_offset = offset; offset += obj->size(); - } while (offset < CardTable::card_size_in_words); + } while (offset < max_offset); if (_scc->get_last_start(index) != prev_offset) { return false; } @@ -542,8 +549,27 @@ ShenandoahScanRemembered::verify_registration(HeapWord* address, return false; } } - } + } else { + // This is a mixed evacuation: rely on mark bits to identify which objects need to be properly registered + // If the object reaching or spanning the end of this card's memory is marked, then last_offset for this card + // should represents this object. Otherwise, last_offset is a don't care. + HeapWord* end_of_interest = base_addr + max_offset; + do { + HeapWord* obj_addr = base_addr + offset; + oop obj = oop(base_addr + offset); + prev_offset = offset; + offset = ctx->get_next_marked_addr(base_addr + offset, end_of_interest) - base_addr; + } while (offset < max_offset); + oop last_obj = oop(base_addr + prev_offset); + if (prev_offset + last_obj->size() >= max_offset) { + if (_scc->get_last_start(index) != prev_offset) { + return false; + } + // otherwise, the value of _scc->get_last_start(index) is a don't care because it represents a dead object and we + // cannot verify its context + } + } return true; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp b/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp index 6a9b0a1759fbc..7379e8c66923f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp @@ -66,6 +66,7 @@ ShenandoahGCSession::ShenandoahGCSession(GCCause::Cause cause, ShenandoahGenerat ); } + ShenandoahGCSession::~ShenandoahGCSession() { _generation->heuristics()->record_cycle_end(); _timer->register_gc_end(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp index d82e3e48b491d..8a9d8ca5ebd3c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp @@ -30,10 +30,12 @@ #include "gc/shenandoah/shenandoahGeneration.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" +#include "gc/shenandoah/shenandoahOldGeneration.hpp" #include "gc/shenandoah/shenandoahRootProcessor.hpp" #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp" #include "gc/shenandoah/shenandoahUtils.hpp" #include "gc/shenandoah/shenandoahVerifier.hpp" +#include "gc/shenandoah/shenandoahYoungGeneration.hpp" #include "memory/allocation.hpp" #include "memory/iterator.inline.hpp" #include "memory/resourceArea.hpp" @@ -345,6 +347,8 @@ class ShenandoahCalculateRegionStatsClosure : public ShenandoahHeapRegionClosure void heap_region_do(ShenandoahHeapRegion* r) { _used += r->used(); + log_debug(gc)("ShenandoahCalculatRegionStatsClosure added " SIZE_FORMAT " for %s Region " SIZE_FORMAT ", yielding: " SIZE_FORMAT, + r->used(), r->is_humongous()? "humongous": "regular", r->index(), _used); _garbage += r->garbage(); _committed += r->is_committed() ? ShenandoahHeapRegion::region_size_bytes() : 0; } @@ -722,6 +726,7 @@ void ShenandoahVerifier::verify_at_safepoint(const char* label, ShenandoahCalculateRegionStatsClosure cl; _heap->heap_region_iterate(&cl); + size_t heap_used = _heap->used(); guarantee(cl.used() == heap_used, "%s: heap used size must be consistent: heap-used = " SIZE_FORMAT "%s, regions-used = " SIZE_FORMAT "%s", @@ -737,6 +742,8 @@ void ShenandoahVerifier::verify_at_safepoint(const char* label, byte_size_in_proper_unit(cl.committed()), proper_unit_for_byte_size(cl.committed())); } + log_debug(gc)("Safepoint verification finished heap usage verification"); + ShenandoahGeneration* generation; if (_heap->mode()->is_generational()) { generation = _heap->active_generation(); @@ -748,14 +755,30 @@ void ShenandoahVerifier::verify_at_safepoint(const char* label, if (generation != NULL) { ShenandoahHeapLocker lock(_heap->lock()); + if (remembered == _verify_remembered_for_marking) { + log_debug(gc)("Safepoint verification of remembered set at mark"); + } else if (remembered == _verify_remembered_for_updating_references) { + log_debug(gc)("Safepoint verification of remembered set at update ref"); + } else if (remembered == _verify_remembered_after_full_gc) { + log_debug(gc)("Safepoint verification of remembered set after full gc"); + } + if (remembered == _verify_remembered_for_marking) { _heap->verify_rem_set_at_mark(); } else if (remembered == _verify_remembered_for_updating_references) { _heap->verify_rem_set_at_update_ref(); + } else if (remembered == _verify_remembered_after_full_gc) { + _heap->verify_rem_set_after_full_gc(); } ShenandoahCalculateRegionStatsClosure cl; generation->heap_region_iterate(&cl); + + log_debug(gc)("Safepoint verification: generation %s usage hereby calculated as: " SIZE_FORMAT, + generation->name(), cl.used()); + log_debug(gc)(" previous tabulation of usage: " SIZE_FORMAT, generation->used()); + + size_t generation_used = generation->used(); guarantee(cl.used() == generation_used, "%s: generation (%s) used size must be consistent: generation-used = " SIZE_FORMAT "%s, regions-used = " SIZE_FORMAT "%s", @@ -764,6 +787,8 @@ void ShenandoahVerifier::verify_at_safepoint(const char* label, byte_size_in_proper_unit(cl.used()), proper_unit_for_byte_size(cl.used())); } + log_debug(gc)("Safepoint verification finished remembered set verification"); + // Internal heap region checks if (ShenandoahVerifyLevel >= 1) { ShenandoahVerifyHeapRegionClosure cl(label, regions); @@ -774,6 +799,8 @@ void ShenandoahVerifier::verify_at_safepoint(const char* label, } } + log_debug(gc)("Safepoint verification finished heap region closure verification"); + OrderAccess::fence(); if (UseTLAB) { @@ -799,6 +826,8 @@ void ShenandoahVerifier::verify_at_safepoint(const char* label, count_reachable = task.processed(); } + log_debug(gc)("Safepoint verification finished getting initial reachable set"); + // Step 3. Walk marked objects. Marked objects might be unreachable. This verifies what collector, // not the application, can see during the region scans. There is no reason to process the objects // that were already verified, e.g. those marked in verification bitmap. There is interaction with TAMS: @@ -816,6 +845,8 @@ void ShenandoahVerifier::verify_at_safepoint(const char* label, guarantee(ShenandoahVerifyLevel < 4 || marked == _verify_marked_incomplete || marked == _verify_marked_disable, "Should be"); } + log_debug(gc)("Safepoint verification finished walking marked objects"); + // Step 4. Verify accumulated liveness data, if needed. Only reliable if verification level includes // marked objects. @@ -849,6 +880,9 @@ void ShenandoahVerifier::verify_at_safepoint(const char* label, } } + log_debug(gc)("Safepoint verification finished accumulation of liveness data"); + + log_info(gc)("Verify %s, Level " INTX_FORMAT " (" SIZE_FORMAT " reachable, " SIZE_FORMAT " marked)", label, ShenandoahVerifyLevel, count_reachable, count_marked); @@ -952,7 +986,7 @@ void ShenandoahVerifier::verify_after_evacuation() { void ShenandoahVerifier::verify_before_updaterefs() { verify_at_safepoint( "Before Updating References", - _verify_remembered_for_updating_references, // do not verify remembered set + _verify_remembered_for_updating_references, // verify read-write remembered set _verify_forwarded_allow, // forwarded references allowed _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well _verify_cset_forwarded, // all cset refs are fully forwarded @@ -1005,6 +1039,20 @@ void ShenandoahVerifier::verify_before_fullgc() { ); } +void ShenandoahVerifier::verify_after_generational_fullgc() { + verify_at_safepoint( + "After Full Generational GC", + _verify_remembered_after_full_gc, // verify read-write remembered set + _verify_forwarded_none, // all objects are non-forwarded + _verify_marked_complete, // all objects are marked in complete bitmap + _verify_cset_none, // no cset references + _verify_liveness_disable, // no reliable liveness data anymore + _verify_regions_notrash_nocset, // no trash, no cset + _verify_gcstate_stable, // full gc cleaned up everything + _verify_all_weak_roots + ); +} + void ShenandoahVerifier::verify_after_fullgc() { verify_at_safepoint( "After Full GC", diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp index 2b6cfda73ac4c..3c72f56b5c871 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp @@ -61,14 +61,18 @@ class ShenandoahVerifier : public CHeapObj { // Disable remembered set verification. _verify_remembered_disable, - // Assure remembered set cards are dirty for every interesting pointer within - // each ShenandoahHeapRegion between bottom() and top(). This is appropriate at - // the init_mark safepoint since all TLABS are retired before we reach this code. + // Assure old objects are registered and remembered set cards within the read-only remembered set are dirty + // for every interesting pointer within each OLD ShenandoahHeapRegion between bottom() and top(). This is + // appropriate at the init_mark safepoint since all TLABS are retired before we reach this code. _verify_remembered_for_marking, - // Assure remembered set cards are dirty for every interesting pointer within - // each ShenandoahHeapRegion between bottom() and get_update_watermark() - _verify_remembered_for_updating_references + // Assure old objects are registered and remembered set cards within the read-write remembered set are dirty + // for every interesting pointer within each OLD ShenandoahHeapRegion between bottom() and top(). + _verify_remembered_for_updating_references, + + // Assure old objects are registered and remembered set cards within the read-write remembered set are dirty + // for every interesting pointer within each OLD ShenandoahHeapRegion between bottom() and top(). + _verify_remembered_after_full_gc } VerifyRememberedSet; typedef enum { @@ -206,6 +210,7 @@ class ShenandoahVerifier : public CHeapObj { void verify_after_updaterefs(); void verify_before_fullgc(); void verify_after_fullgc(); + void verify_after_generational_fullgc(); void verify_after_degenerated(); void verify_generic(VerifyOption option); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp index 9d7ee3810ba96..cf27c899acdf9 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp @@ -86,21 +86,6 @@ void ShenandoahYoungGeneration::promote_tenured_regions() { log_info(gc)("Promoted " SIZE_FORMAT " regions.", task._promoted); } -void ShenandoahYoungGeneration::promote_all_regions() { - // This only happens on a full stw collect. No allocations can happen here. - shenandoah_assert_safepoint(); - - ShenandoahHeap* heap = ShenandoahHeap::heap(); - for (size_t index = 0; index < heap->num_regions(); index++) { - ShenandoahHeapRegion* r = heap->get_region(index); - if (r->is_young()) { - r->promote(true); - } - } - assert(_affiliated_region_count == 0, "young generation must not have affiliated regions after reset"); - _used = 0; -} - bool ShenandoahYoungGeneration::contains(ShenandoahHeapRegion* region) const { // TODO: why not test for equals YOUNG_GENERATION? As written, returns true for regions that are FREE return region->affiliation() != OLD_GENERATION; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp index 6241f7f4cfb5c..70ce349155a4d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp @@ -45,8 +45,6 @@ class ShenandoahYoungGeneration : public ShenandoahGeneration { bool contains(oop obj) const override; void promote_tenured_regions(); - void promote_all_regions(); - void set_old_gen_task_queues(ShenandoahObjToScanQueueSet* old_gen_queues) { _old_gen_task_queues = old_gen_queues; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp index e746dd2b42f17..0f25ef02ae387 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp @@ -98,9 +98,10 @@ range(0,100) \ \ product(uintx, ShenandoahMinFreeThreshold, 10, EXPERIMENTAL, \ - "How much heap should be free before most heuristics trigger the "\ - "collection, even without other triggers. Provides the safety " \ - "margin for many heuristics. In percents of (soft) max heap size.")\ + "Percentage of free heap memory below which most heuristics " \ + "trigger collection independent of other triggers. Provides " \ + "a safety margin for many heuristics. In percents of (soft) " \ + "max heap size.") \ range(0,100) \ \ product(uintx, ShenandoahAllocationThreshold, 0, EXPERIMENTAL, \ From 0a72c50d2847b387135a92a3ac3b6a87bf88d308 Mon Sep 17 00:00:00 2001 From: Aleksey Shipilev Date: Thu, 2 Sep 2021 07:37:31 +0000 Subject: [PATCH 060/254] Fix ShenandoahFullGC worker stats handling Reviewed-by: zgu --- src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index 3a2dc5b3698b6..2931ba8b31b40 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -1070,9 +1070,12 @@ void ShenandoahFullGC::phase2_calculate_target_addresses(ShenandoahHeapRegionSet distribute_slices(worker_slices); - size_t num_workers = heap->workers()->total_workers(); - size_t old_used[num_workers]; - size_t young_used[num_workers]; + size_t num_workers = heap->max_workers(); + + ResourceMark rm; + size_t* old_used = NEW_RESOURCE_ARRAY(size_t, num_workers); + size_t* young_used = NEW_RESOURCE_ARRAY(size_t, num_workers); + ShenandoahPrepareForCompactionTask task(_preserved_marks, worker_slices, num_workers, old_used, young_used); heap->workers()->run_task(&task); @@ -1082,6 +1085,9 @@ void ShenandoahFullGC::phase2_calculate_target_addresses(ShenandoahHeapRegionSet heap->young_generation()->increase_used(task.young_used()); heap->old_generation()->increase_used(task.old_used()); } + + FREE_RESOURCE_ARRAY(size_t, old_used, num_workers); + FREE_RESOURCE_ARRAY(size_t, young_used, num_workers); } // Compute the new addresses for humongous objects From f01e568c5ba92bdd2ac023fe73c16bd714db8e04 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Thu, 2 Sep 2021 15:37:46 +0000 Subject: [PATCH 061/254] Fix misuse of atomic flag Reviewed-by: zgu, shade --- .../share/gc/shenandoah/shenandoahControlThread.cpp | 4 ++-- src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp | 13 +++++++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index f0b6731cd7440..dc0684d017699 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -522,7 +522,7 @@ void ShenandoahControlThread::resume_concurrent_old_cycle(ShenandoahGeneration* if (heap->cancelled_gc()) { // It's possible the gc cycle was cancelled after the last time // the collection checked for cancellation. In which case, the - // old gc cycle is still completed and we have to deal with this + // old gc cycle is still completed, and we have to deal with this // cancellation. We set the degeneration point to be outside // the cycle because if this is an allocation failure, that is // what must be done (there is no degenerated old cycle). If the @@ -746,7 +746,7 @@ void ShenandoahControlThread::notify_control_thread() { } bool ShenandoahControlThread::preempt_old_marking(GenerationMode generation) { - return generation == YOUNG && _allow_old_preemption.is_set(); + return generation == YOUNG && _allow_old_preemption.try_unset(); } void ShenandoahControlThread::handle_requested_gc(GCCause::Cause cause) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp index 03e7608692980..2e6faedb2e05c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp @@ -120,8 +120,17 @@ bool ShenandoahOldGC::collect(GCCause::Cause cause) { // Continue concurrent mark, do not reset regions, do not mark roots, do not collect $200. _allow_preemption.set(); entry_mark(); - _allow_preemption.unset(); - if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark)) return false; + if (!_allow_preemption.try_unset()) { + // The regulator thread has unset the preemption guard. That thread will shortly cancel + // the gc, but the control thread is now racing it. Wait until this thread sees the cancellation. + while (!heap->cancelled_gc()) { + SpinPause(); + } + } + + if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark)) { + return false; + } // Complete marking under STW vmop_entry_final_mark(); From aa7ff7c1102a1813486e74091f126f4eee103679 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Thu, 9 Sep 2021 21:13:50 +0000 Subject: [PATCH 062/254] Enable remembered set verification during global collections Reviewed-by: kdnilsen --- .../gc/shenandoah/shenandoahConcurrentGC.cpp | 45 ++++++++++++++- .../gc/shenandoah/shenandoahConcurrentGC.hpp | 2 + .../share/gc/shenandoah/shenandoahHeap.cpp | 56 +++++++------------ .../gc/shenandoah/shenandoahHeapRegion.cpp | 4 +- .../gc/shenandoah/shenandoahMark.inline.hpp | 21 +++---- .../share/gc/shenandoah/shenandoahOldGC.cpp | 2 - .../shenandoah/shenandoahScanRemembered.hpp | 3 +- .../shenandoahScanRemembered.inline.hpp | 49 ++++++++-------- 8 files changed, 99 insertions(+), 83 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index 272f5198348b7..f3de4fb9cccf7 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -72,6 +72,20 @@ class ShenandoahBreakpointMarkScope : public StackObj { } }; +class ShenandoahGlobalCoalesceAndFill : public ShenandoahHeapRegionClosure { + public: + virtual void heap_region_do(ShenandoahHeapRegion* region) override { + // old region is not in the collection set and was not immediately trashed + if (region->is_old() && region->is_active() && !region->is_humongous()) { + region->oop_fill_and_coalesce(); + } + } + + virtual bool is_thread_safe() override { + return true; + } +}; + ShenandoahConcurrentGC::ShenandoahConcurrentGC(ShenandoahGeneration* generation, bool do_old_gc_bootstrap) : _mark(generation), _degen_point(ShenandoahDegenPoint::_degenerated_unset), @@ -150,6 +164,10 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) { entry_strong_roots(); } + if (heap->mode()->is_generational() && _generation->generation_mode() == GLOBAL) { + entry_global_coalesce_and_fill(); + } + // Continue the cycle with evacuation and optional update-refs. // This may be skipped if there is nothing to evacuate. // If so, evac_in_progress would be unset by collection set preparation code. @@ -239,9 +257,11 @@ void ShenandoahConcurrentGC::entry_init_mark() { ShenandoahWorkerPolicy::calc_workers_for_init_marking(), "init marking"); - if (ShenandoahHeap::heap()->mode()->is_generational() && (_generation->generation_mode() == YOUNG)) { + if (ShenandoahHeap::heap()->mode()->is_generational() + && (_generation->generation_mode() == YOUNG || (_generation->generation_mode() == GLOBAL && ShenandoahVerify))) { // The current implementation of swap_remembered_set() copies the write-card-table - // to the read-card-table. + // to the read-card-table. The remembered sets are also swapped for GLOBAL collections + // so that the verifier works with the correct copy of the card table when verifying. _generation->swap_remembered_set(); } @@ -479,6 +499,21 @@ void ShenandoahConcurrentGC::entry_cleanup_complete() { op_cleanup_complete(); } +void ShenandoahConcurrentGC::entry_global_coalesce_and_fill() { + ShenandoahHeap* const heap = ShenandoahHeap::heap(); + + const char* msg = "Coalescing and filling old regions in global collect"; + ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::coalesce_and_fill); + + TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); + EventMark em("%s", msg); + ShenandoahWorkerScope scope(heap->workers(), + ShenandoahWorkerPolicy::calc_workers_for_conc_marking(), + "concurrent coalesce and fill"); + + op_global_coalesce_and_fill(); +} + void ShenandoahConcurrentGC::op_reset() { ShenandoahHeap* const heap = ShenandoahHeap::heap(); if (ShenandoahPacing) { @@ -1027,6 +1062,12 @@ void ShenandoahConcurrentGC::op_cleanup_complete() { ShenandoahHeap::heap()->free_set()->recycle_trash(); } +void ShenandoahConcurrentGC::op_global_coalesce_and_fill() { + ShenandoahHeap* const heap = ShenandoahHeap::heap(); + ShenandoahGlobalCoalesceAndFill coalesce; + heap->parallel_heap_region_iterate(&coalesce); +} + bool ShenandoahConcurrentGC::check_cancellation_and_abort(ShenandoahDegenPoint point) { if (ShenandoahHeap::heap()->cancelled_gc()) { _degen_point = point; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp index 9699bade54d06..57957a5378bcd 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp @@ -101,6 +101,7 @@ class ShenandoahConcurrentGC : public ShenandoahGC { void entry_update_thread_roots(); void entry_updaterefs(); void entry_cleanup_complete(); + void entry_global_coalesce_and_fill(); // Actual work for the phases void op_reset(); @@ -120,6 +121,7 @@ class ShenandoahConcurrentGC : public ShenandoahGC { void op_final_updaterefs(); void op_final_roots(); void op_cleanup_complete(); + void op_global_coalesce_and_fill(); // Messages for GC trace events, they have to be immortal for // passing around the logging/tracing systems diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 6d4c324891c48..38f4f8a154083 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -2199,12 +2199,7 @@ class ShenandoahUpdateHeapRefsTask : public AbstractGangTask { _heap->marked_object_oop_iterate(r, &cl, update_watermark); } else if (r->affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) { if (_heap->active_generation()->generation_mode() == GLOBAL) { - // This code is only relevant to GLOBAL GC. With OLD GC, all coalescing and filling is done before any relevant - // evacuations. - - // This is an old region in a global cycle. Make sure that the next cycle does not iterate over dead objects - // which haven't had their references updated. This is not a promotion. - r->global_oop_iterate_and_fill_dead(&cl); + _heap->marked_object_oop_iterate(r, &cl, update_watermark); } else { // Old region in a young cycle or mixed cycle. if (ShenandoahUseSimpleCardScanning) { @@ -2595,15 +2590,14 @@ void ShenandoahHeap::verify_rem_set_at_mark() { assert(mode()->is_generational(), "Only verify remembered set for generational operational modes"); ShenandoahRegionIterator iterator; - ShenandoahMarkingContext* mark_context = marking_context(); RememberedScanner* scanner = card_scan(); ShenandoahVerifyRemSetClosure check_interesting_pointers(true); ShenandoahMarkingContext* ctx; log_debug(gc)("Verifying remembered set at %s mark", doing_mixed_evacuations()? "mixed": "young"); - if (doing_mixed_evacuations()) { - ctx = mark_context; + if (doing_mixed_evacuations() || active_generation()->generation_mode() == GLOBAL) { + ctx = complete_marking_context(); } else { ctx = nullptr; } @@ -2612,7 +2606,7 @@ void ShenandoahHeap::verify_rem_set_at_mark() { ShenandoahHeapRegion* r = iterator.next(); if (r == nullptr) break; - if (r->is_old()) { + if (r->is_old() && r->is_active()) { HeapWord* obj_addr = r->bottom(); if (r->is_humongous_start()) { oop obj = cast_to_oop(obj_addr); @@ -2626,15 +2620,15 @@ void ShenandoahHeap::verify_rem_set_at_mark() { // else, object's start is marked dirty and obj is not an objArray, so any interesting pointers are covered } // else, this humongous object is not marked so no need to verify its internal pointers - if (!scanner->verify_registration(obj_addr, obj->size())) { + if (!scanner->verify_registration(obj_addr, ctx)) { ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, obj_addr, NULL, "Verify init-mark remembered set violation", "object not properly registered", __FILE__, __LINE__); } } else if (!r->is_humongous()) { - HeapWord* t = r->top(); - while (obj_addr < t) { + HeapWord* top = r->top(); + while (obj_addr < top) { oop obj = cast_to_oop(obj_addr); - // ctx->is_marked() returns true if mark bit set (TAMS not relevant here) + // ctx->is_marked() returns true if mark bit set (TAMS not relevant during init mark) if (!ctx || ctx->is_marked(obj)) { // For regular objects (not object arrays), if the card holding the start of the object is dirty, // we do not need to verify that cards spanning interesting pointers within this object are dirty. @@ -2642,18 +2636,15 @@ void ShenandoahHeap::verify_rem_set_at_mark() { obj->oop_iterate(&check_interesting_pointers); } // else, object's start is marked dirty and obj is not an objArray, so any interesting pointers are covered - if (!scanner->verify_registration(obj_addr, obj->size())) { + if (!scanner->verify_registration(obj_addr, ctx)) { ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, obj_addr, NULL, "Verify init-mark remembered set violation", "object not properly registered", __FILE__, __LINE__); } obj_addr += obj->size(); - } // Else, this object is not live so we don't verify dirty cards contained therein. - - if (ctx) { - // TAMS not relevant here - obj_addr = ctx->get_next_marked_addr(obj_addr, t); } else { - obj_addr += obj->size(); + // This object is not live so we don't verify dirty cards contained therein + assert(ctx->top_at_mark_start(r) == top, "Expect tams == top at start of mark."); + obj_addr = ctx->get_next_marked_addr(obj_addr, top); } } } // else, we ignore humongous continuation region @@ -2681,7 +2672,7 @@ void ShenandoahHeap::help_verify_region_rem_set(ShenandoahHeapRegion* r, Shenand } // else, this humongous object is not live so no need to verify its internal pointers - if ((obj_addr < registration_watermark) && !scanner->verify_registration(obj_addr, obj->size())) { + if ((obj_addr < registration_watermark) && !scanner->verify_registration(obj_addr, ctx)) { ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, obj_addr, NULL, message, "object not properly registered", __FILE__, __LINE__); } @@ -2698,22 +2689,15 @@ void ShenandoahHeap::help_verify_region_rem_set(ShenandoahHeapRegion* r, Shenand } // else, object's start is marked dirty and obj is not an objArray, so any interesting pointers are covered - if ((obj_addr < registration_watermark) && !scanner->verify_registration(obj_addr, obj->size())) { + if ((obj_addr < registration_watermark) && !scanner->verify_registration(obj_addr, ctx)) { ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, obj_addr, NULL, message, "object not properly registered", __FILE__, __LINE__); } - } // Else, this object is not live so we don't verify dirty cards contained therein. - - if (ctx) { - ShenandoahHeapRegion* r = heap_region_containing(obj_addr); - HeapWord* tams = ctx->top_at_mark_start(r); - if (obj_addr >= tams) { - obj_addr += obj->size(); - } else { - obj_addr = ctx->get_next_marked_addr(obj_addr, tams); - } - } else { obj_addr += obj->size(); + } else { + // This object is not live so we don't verify dirty cards contained therein + HeapWord* tams = ctx->top_at_mark_start(r); + obj_addr = ctx->get_next_marked_addr(obj_addr, tams); } } } @@ -2746,8 +2730,8 @@ void ShenandoahHeap::verify_rem_set_at_update_ref() { ShenandoahRegionIterator iterator; ShenandoahMarkingContext* ctx; - if (doing_mixed_evacuations()) { - ctx = marking_context(); + if (doing_mixed_evacuations() || active_generation()->generation_mode() == GLOBAL) { + ctx = complete_marking_context(); } else { ctx = nullptr; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index 2137f080095fb..8d0c00278dbcd 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -441,9 +441,7 @@ void ShenandoahHeapRegion::oop_fill_and_coalesce() { // explicitly marked. HeapWord* t = marking_context->top_at_mark_start(this); - // Expect this to be invoked only from within threads perfoming old-gen GC, and expect - // old-gen marking to be completed before these threads invoke this service. - assert(heap->active_generation()->generation_mode() == OLD, "sanity"); + // Expect marking to be completed before these threads invoke this service. assert(heap->active_generation()->is_mark_complete(), "sanity"); while (obj_addr < t) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp index e548f1e2a5b65..bb05ccb44beaa 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp @@ -272,27 +272,24 @@ inline void ShenandoahMark::mark_through_ref(T *p, ShenandoahObjToScanQueue* q, if (!CompressedOops::is_null(o)) { oop obj = CompressedOops::decode_not_null(o); + ShenandoahHeap* heap = ShenandoahHeap::heap(); shenandoah_assert_not_forwarded(p, obj); - shenandoah_assert_not_in_cset_except(p, obj, ShenandoahHeap::heap()->cancelled_gc()); + shenandoah_assert_not_in_cset_except(p, obj, heap->cancelled_gc()); if (in_generation(obj)) { mark_ref(q, mark_context, weak, obj); shenandoah_assert_marked(p, obj); - if (ShenandoahHeap::heap()->mode()->is_generational()) { + if (heap->mode()->is_generational()) { // TODO: As implemented herein, GLOBAL collections reconstruct the card table during GLOBAL concurrent // marking. Note that the card table is cleaned at init_mark time so it needs to be reconstructed to support // future young-gen collections. It might be better to reconstruct card table in // ShenandoahHeapRegion::global_oop_iterate_and_fill_dead. We could either mark all live memory as dirty, or could // use the GLOBAL update-refs scanning of pointers to determine precisely which cards to flag as dirty. - // - if ((GENERATION == YOUNG) && ShenandoahHeap::heap()->is_in(p) && ShenandoahHeap::heap()->is_in_old(p)) { - RememberedScanner* scanner = ShenandoahHeap::heap()->card_scan(); + if (GENERATION == YOUNG && heap->is_in_old(p)) { // Mark card as dirty because remembered set scanning still finds interesting pointer. - ShenandoahHeap::heap()->mark_card_as_dirty((HeapWord*)p); - } else if ((GENERATION == GLOBAL) && in_generation(obj) && - ShenandoahHeap::heap()->is_in(p) && ShenandoahHeap::heap()->is_in_old(p)) { - RememberedScanner* scanner = ShenandoahHeap::heap()->card_scan(); + heap->mark_card_as_dirty((HeapWord*)p); + } else if (GENERATION == GLOBAL && heap->is_in_old(p) && heap->is_in_young(obj)) { // Mark card as dirty because GLOBAL marking finds interesting pointer. - ShenandoahHeap::heap()->mark_card_as_dirty((HeapWord*)p); + heap->mark_card_as_dirty((HeapWord*)p); } } } else if (old != nullptr) { @@ -303,8 +300,8 @@ inline void ShenandoahMark::mark_through_ref(T *p, ShenandoahObjToScanQueue* q, // Old mark, found a young pointer. // TODO: Rethink this: may be redundant with dirtying of cards identified during young-gen remembered set scanning // and by mutator write barriers. Assert - assert(ShenandoahHeap::heap()->is_in_young(obj), "Expected young object."); - ShenandoahHeap::heap()->mark_card_as_dirty(p); + assert(heap->is_in_young(obj), "Expected young object."); + heap->mark_card_as_dirty(p); } } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp index c56d5a448ea5b..c238d572d685c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp @@ -51,8 +51,6 @@ class ShenandoahConcurrentCoalesceAndFillTask : public AbstractGangTask { } void work(uint worker_id) { - ShenandoahHeap* heap = ShenandoahHeap::heap(); - for (uint region_idx = worker_id; region_idx < _coalesce_and_fill_region_count; region_idx += _nworkers) { ShenandoahHeapRegion* r = _coalesce_and_fill_region_array[region_idx]; if (!r->is_humongous()) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp index a296f842fefe4..1ff4f80fb13ac 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp @@ -220,6 +220,7 @@ class ShenandoahReferenceProcessor; class ShenandoahConcurrentMark; class ShenandoahHeap; class ShenandoahRegionIterator; +class ShenandoahMarkingContext; class CardTable; @@ -952,7 +953,7 @@ class ShenandoahScanRemembered: public CHeapObj { void coalesce_objects(HeapWord *addr, size_t length_in_words); // Return true iff this object is "properly" registered. - bool verify_registration(HeapWord* address, size_t size_in_words); + bool verify_registration(HeapWord* address, ShenandoahMarkingContext* ctx); // clear the cards to clean, and clear the object_starts info to no objects void mark_range_as_empty(HeapWord *addr, size_t length_in_words); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp index 283b670862f3e..70241e3320b3b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp @@ -469,7 +469,7 @@ ShenandoahScanRemembered::register_object_wo_lock(HeapWord *addr) template inline bool -ShenandoahScanRemembered::verify_registration(HeapWord* address, size_t size_in_words) { +ShenandoahScanRemembered::verify_registration(HeapWord* address, ShenandoahMarkingContext* ctx) { size_t index = card_index_for_addr(address); if (!_scc->has_object(index)) { @@ -478,13 +478,6 @@ ShenandoahScanRemembered::verify_registration(HeapWord* address, HeapWord* base_addr = addr_for_card_index(index); size_t offset = _scc->get_first_start(index); ShenandoahHeap* heap = ShenandoahHeap::heap(); - ShenandoahMarkingContext* ctx; - - if (heap->doing_mixed_evacuations()) { - ctx = heap->marking_context(); - } else { - ctx = nullptr; - } // Verify that I can find this object within its enclosing card by scanning forward from first_start. while (base_addr + offset < address) { @@ -493,13 +486,9 @@ ShenandoahScanRemembered::verify_registration(HeapWord* address, offset += obj->size(); } else { // If this object is not live, don't trust its size(); all objects above tams are live. - ShenandoahHeapRegion* r = heap->heap_region_containing(base_addr + offset); + ShenandoahHeapRegion* r = heap->heap_region_containing(obj); HeapWord* tams = ctx->top_at_mark_start(r); - if (base_addr + offset >= tams) { - offset += obj->size(); - } else { - offset = ctx->get_next_marked_addr(base_addr + offset, tams) - base_addr; - } + offset = ctx->get_next_marked_addr(base_addr + offset, tams) - base_addr; } } if (base_addr + offset != address){ @@ -521,7 +510,6 @@ ShenandoahScanRemembered::verify_registration(HeapWord* address, size_t prev_offset; if (!ctx) { do { - HeapWord* obj_addr = base_addr + offset; oop obj = cast_to_oop(base_addr + offset); prev_offset = offset; offset += obj->size(); @@ -537,10 +525,12 @@ ShenandoahScanRemembered::verify_registration(HeapWord* address, // cannot use card_index_for_addr(base_addr + offset) because it asserts arg < end of whole heap size_t end_card_index = index + offset / CardTable::card_size_in_words; - // If there is a following object registered, it should begin where this object ends. - if ((base_addr + offset < _rs->whole_heap_end()) && _scc->has_object(end_card_index) && - ((addr_for_card_index(end_card_index) + _scc->get_first_start(end_card_index)) != (base_addr + offset))) { - return false; + if (end_card_index > index) { + // If there is a following object registered on the next card, it should begin where this object ends. + if ((base_addr + offset < _rs->whole_heap_end()) && _scc->has_object(end_card_index) && + ((addr_for_card_index(end_card_index) + _scc->get_first_start(end_card_index)) != (base_addr + offset))) { + return false; + } } // Assure that no other objects are registered "inside" of this one. @@ -550,17 +540,22 @@ ShenandoahScanRemembered::verify_registration(HeapWord* address, } } } else { - // This is a mixed evacuation: rely on mark bits to identify which objects need to be properly registered - + // This is a mixed evacuation or a global collect: rely on mark bits to identify which objects need to be properly registered + assert(!ShenandoahHeap::heap()->is_concurrent_old_mark_in_progress(), "Cannot rely on mark context here."); // If the object reaching or spanning the end of this card's memory is marked, then last_offset for this card - // should represents this object. Otherwise, last_offset is a don't care. - HeapWord* end_of_interest = base_addr + max_offset; + // should represent this object. Otherwise, last_offset is a don't care. + ShenandoahHeapRegion* region = heap->heap_region_containing(base_addr + offset); + HeapWord* tams = ctx->top_at_mark_start(region); do { - HeapWord* obj_addr = base_addr + offset; - oop obj = cast_to_oop(base_addr + offset); prev_offset = offset; - offset = ctx->get_next_marked_addr(base_addr + offset, end_of_interest) - base_addr; - } while (offset < max_offset); + oop obj = cast_to_oop(base_addr + offset); + if (ctx->is_marked(obj)) { + offset += obj->size(); + } else { + offset = ctx->get_next_marked_addr(base_addr + offset, tams) - base_addr; + // offset will be zero if no objects are marked in this card. + } + } while (offset > 0 && offset < max_offset); oop last_obj = cast_to_oop(base_addr + prev_offset); if (prev_offset + last_obj->size() >= max_offset) { if (_scc->get_last_start(index) != prev_offset) { From 40174b8f5be10d991c8e322b2ad38b954f769779 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Fri, 10 Sep 2021 18:28:47 +0000 Subject: [PATCH 063/254] Remove log message that could access an out of bounds region Reviewed-by: kdnilsen --- src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 38f4f8a154083..ac60d6e56b241 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -1260,12 +1260,6 @@ class ShenandoahRetireGCLABClosure : public ThreadClosure { // the update-refs processing that follows. After the updating of old-gen references is done, it is ok to carve // this remnant object into smaller pieces during the subsequent evacuation pass, as long as the PLAB is made parsable // again before the next update-refs phase. - if (plab->top() != nullptr) { - ShenandoahHeapRegion* r = ShenandoahHeap::heap()->heap_region_containing(plab->top()); - log_debug(gc)("Retiring plab with top: " PTR_FORMAT ", hard_end: " PTR_FORMAT - " + AlignmentReserve, in %s Region " SIZE_FORMAT, - p2i(plab->top()), p2i(plab->top() + plab->words_remaining()), affiliation_name(r->affiliation()), r->index()); - } // else, don't bother to report retirement ShenandoahHeap::heap()->retire_plab(plab); if (_resize && ShenandoahThreadLocalData::plab_size(thread) > 0) { ShenandoahThreadLocalData::set_plab_size(thread, 0); From dd7af296d20dca914fc2f39171a852f8408d6a5a Mon Sep 17 00:00:00 2001 From: William Kemper Date: Wed, 15 Sep 2021 17:09:06 +0000 Subject: [PATCH 064/254] Fix handling of weak references during verification Reviewed-by: kdnilsen, rkennke --- .../gc/shenandoah/shenandoahRootVerifier.cpp | 8 +++---- .../gc/shenandoah/shenandoahRootVerifier.hpp | 4 ++-- .../shenandoah/shenandoahScanRemembered.hpp | 1 - .../shenandoahScanRemembered.inline.hpp | 21 ------------------- .../gc/shenandoah/shenandoahVerifier.cpp | 4 ++-- 5 files changed, 8 insertions(+), 30 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.cpp b/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.cpp index db7bed9f17f4a..8665499b78b53 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.cpp @@ -53,7 +53,7 @@ ShenandoahGCStateResetter::~ShenandoahGCStateResetter() { assert(_heap->gc_state() == _gc_state, "Should be restored"); } -void ShenandoahRootVerifier::roots_do(OopClosure* oops) { +void ShenandoahRootVerifier::roots_do(OopIterateClosure* oops) { ShenandoahGCStateResetter resetter; shenandoah_assert_safepoint(); @@ -70,7 +70,7 @@ void ShenandoahRootVerifier::roots_do(OopClosure* oops) { ShenandoahHeap* heap = ShenandoahHeap::heap(); if (heap->mode()->is_generational() && heap->is_gc_generation_young()) { shenandoah_assert_safepoint(); - heap->card_scan()->oops_do(oops); + heap->card_scan()->roots_do(oops); } // Do thread roots the last. This allows verification code to find @@ -79,7 +79,7 @@ void ShenandoahRootVerifier::roots_do(OopClosure* oops) { Threads::possibly_parallel_oops_do(true, oops, NULL); } -void ShenandoahRootVerifier::strong_roots_do(OopClosure* oops) { +void ShenandoahRootVerifier::strong_roots_do(OopIterateClosure* oops) { ShenandoahGCStateResetter resetter; shenandoah_assert_safepoint(); @@ -92,7 +92,7 @@ void ShenandoahRootVerifier::strong_roots_do(OopClosure* oops) { ShenandoahHeap* heap = ShenandoahHeap::heap(); if (heap->mode()->is_generational() && heap->is_gc_generation_young()) { - heap->card_scan()->oops_do(oops); + heap->card_scan()->roots_do(oops); } // Do thread roots the last. This allows verification code to find diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.hpp b/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.hpp index 54c95512a9ce5..d2715f4a3dfd2 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.hpp @@ -41,8 +41,8 @@ class ShenandoahGCStateResetter : public StackObj { class ShenandoahRootVerifier : public AllStatic { public: // Used to seed ShenandoahVerifier, do not honor root type filter - static void roots_do(OopClosure* cl); - static void strong_roots_do(OopClosure* cl); + static void roots_do(OopIterateClosure* cl); + static void strong_roots_do(OopIterateClosure* cl); }; #endif // SHARE_GC_SHENANDOAH_SHENANDOAHROOTVERIFIER_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp index 1ff4f80fb13ac..1e0aabcd072d0 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp @@ -1023,7 +1023,6 @@ class ShenandoahScanRemembered: public CHeapObj { // implementations will want to update this value each time they // cross one of these boundaries. void roots_do(OopIterateClosure* cl); - void oops_do(OopClosure* cl); }; typedef ShenandoahScanRemembered RememberedScanner; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp index 70241e3320b3b..b921820cfda62 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp @@ -810,27 +810,6 @@ ShenandoahScanRemembered::cluster_for_addr(HeapWordImpl **addr) { return result; } -class ShenandoahOopIterateAdapter : public BasicOopIterateClosure { - private: - OopClosure* _cl; - public: - explicit ShenandoahOopIterateAdapter(OopClosure* cl) : _cl(cl) {} - - void do_oop(oop* o) { - _cl->do_oop(o); - } - - void do_oop(narrowOop* o) { - _cl->do_oop(o); - } -}; - -template -inline void ShenandoahScanRemembered::oops_do(OopClosure* cl) { - ShenandoahOopIterateAdapter adapter(cl); - roots_do(&adapter); -} - template inline void ShenandoahScanRemembered::roots_do(OopIterateClosure* cl) { ShenandoahHeap* heap = ShenandoahHeap::heap(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp index 04ee9b00404ac..1ead9cb228320 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp @@ -1052,7 +1052,7 @@ void ShenandoahVerifier::verify_after_fullgc() { ); } -class ShenandoahVerifyNoForwared : public OopClosure { +class ShenandoahVerifyNoForwared : public BasicOopIterateClosure { private: template void do_oop_work(T* p) { @@ -1072,7 +1072,7 @@ class ShenandoahVerifyNoForwared : public OopClosure { void do_oop(oop* p) { do_oop_work(p); } }; -class ShenandoahVerifyInToSpaceClosure : public OopClosure { +class ShenandoahVerifyInToSpaceClosure : public BasicOopIterateClosure { private: template void do_oop_work(T* p) { From 6a802c24caefcf555b381674bd81d8e5c6d25b0f Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Wed, 15 Sep 2021 17:41:45 +0000 Subject: [PATCH 065/254] Fix full gc with class unloading Reviewed-by: rkennke --- .../gc/shenandoah/shenandoahControlThread.cpp | 52 +++++++++++-------- .../gc/shenandoah/shenandoahControlThread.hpp | 5 +- .../gc/shenandoah/shenandoahDegeneratedGC.cpp | 19 +++++-- .../gc/shenandoah/shenandoahDegeneratedGC.hpp | 3 ++ 4 files changed, 50 insertions(+), 29 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index 5ba556b1bbff6..e2f5373f5e1ce 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -150,6 +150,7 @@ void ShenandoahControlThread::run_service() { } else { heuristics->record_allocation_failure_gc(); policy->record_alloc_failure_to_full(); + generation = GLOBAL; set_gc_mode(stw_full); } } else if (explicit_gc_requested) { @@ -251,7 +252,10 @@ void ShenandoahControlThread::run_service() { break; } case stw_degenerated: { - service_stw_degenerated_cycle(cause, degen_point); + if (!service_stw_degenerated_cycle(cause, degen_point)) { + // The degenerated GC was upgraded to a Full GC + generation = GLOBAL; + } break; } case stw_full: { @@ -307,8 +311,7 @@ void ShenandoahControlThread::run_service() { // Clear metaspace oom flag, if current cycle unloaded classes if (heap->unload_classes()) { - // HEY! Should we do this if the cycle was cancelled/degenerated? - assert(generation != YOUNG, "should not unload classes in young cycle"); + assert(generation == GLOBAL, "Only unload classes during GLOBAL cycle"); global_heuristics->clear_metaspace_oom(); } @@ -386,27 +389,26 @@ void ShenandoahControlThread::run_service() { // and explicit GC requests are handled by the controller thread and always // run a global cycle (which is concurrent by default, but may be overridden // by command line options). Old cycles always degenerate to a global cycle. -// Young cycles are degenerated to complete the young cycle. +// Young cycles are degenerated to complete the young cycle. Young +// and old degen may upgrade to Full GC. Full GC may also be +// triggered directly by a System.gc() invocation. // // -// +-------------+----------+ Idle +--------------+ -// | | + | -// | | | | -// | | | | -// | | | | -// | v v v -// | -// | Young <---+ Resume Old <----+ Bootstrap Old -// | + + + + -// | | | | | -// | v | | v -// v Young Degen | | Young Degen -// | | -// Global <--------------------+ | -// + | -// | v -// | -// +---------------------> Global Degen +// +-----+ Idle +-----+-----------+---------------------+ +// | + | | | +// | | | | | +// | | v | | +// | | Bootstrap Old +-- | ------------+ | +// | | + | | | +// | | | | | | +// | v v v v | +// | Resume Old <----------+ Young +--> Young Degen | +// | + + + | +// v | | | | +// Global <-+ | | | +// + | | | +// | v v | +// +---> Global Degen +--------------------> Full <----+ // void ShenandoahControlThread::service_concurrent_normal_cycle( const ShenandoahHeap* heap, const GenerationMode generation, GCCause::Cause cause) { @@ -585,6 +587,9 @@ void ShenandoahControlThread::service_concurrent_cycle(ShenandoahGeneration* gen } else { assert(heap->cancelled_gc(), "Must have been cancelled"); check_cancellation_or_degen(gc.degen_point()); + assert(generation->generation_mode() != OLD, "Old GC takes a different control path"); + // Concurrent young-gen collection degenerates to young + // collection. Same for global collections. _degen_generation = generation; } } @@ -642,7 +647,7 @@ void ShenandoahControlThread::service_stw_full_cycle(GCCause::Cause cause) { heap->shenandoah_policy()->record_success_full(); } -void ShenandoahControlThread::service_stw_degenerated_cycle(GCCause::Cause cause, ShenandoahGC::ShenandoahDegenPoint point) { +bool ShenandoahControlThread::service_stw_degenerated_cycle(GCCause::Cause cause, ShenandoahGC::ShenandoahDegenPoint point) { assert (point != ShenandoahGC::_degenerated_unset, "Degenerated point should be set"); ShenandoahHeap* const heap = ShenandoahHeap::heap(); @@ -662,6 +667,7 @@ void ShenandoahControlThread::service_stw_degenerated_cycle(GCCause::Cause cause _degen_generation->heuristics()->record_success_degenerated(); heap->shenandoah_policy()->record_success_degenerated(); + return !gc.upgraded_to_full(); } void ShenandoahControlThread::service_uncommit(double shrink_before, size_t shrink_until) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp index 064da5623c480..78180e3078a5e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp @@ -105,7 +105,10 @@ class ShenandoahControlThread: public ConcurrentGCThread { void resume_concurrent_old_cycle(ShenandoahGeneration* generation, GCCause::Cause cause); void service_concurrent_cycle(ShenandoahGeneration* generation, GCCause::Cause cause, bool reset_old_bitmap_specially); void service_stw_full_cycle(GCCause::Cause cause); - void service_stw_degenerated_cycle(GCCause::Cause cause, ShenandoahGC::ShenandoahDegenPoint point); + + // Return true if degenerated cycle finishes normally. Return false if the degenerated cycle transformed itself + // into a full GC. + bool service_stw_degenerated_cycle(GCCause::Cause cause, ShenandoahGC::ShenandoahDegenPoint point); void service_uncommit(double shrink_before, size_t shrink_until); bool try_set_alloc_failure_gc(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp index 2d7adc641158a..08116621696cf 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp @@ -46,7 +46,8 @@ ShenandoahDegenGC::ShenandoahDegenGC(ShenandoahDegenPoint degen_point, ShenandoahGeneration* generation) : ShenandoahGC(), _degen_point(degen_point), - _generation(generation) { + _generation(generation), + _upgraded_to_full(false) { } bool ShenandoahDegenGC::collect(GCCause::Cause cause) { @@ -331,15 +332,13 @@ void ShenandoahDegenGC::op_cleanup_complete() { } void ShenandoahDegenGC::op_degenerated_fail() { - log_info(gc)("Cannot finish degeneration, upgrading to Full GC"); - ShenandoahHeap::heap()->shenandoah_policy()->record_degenerated_upgrade_to_full(); - + upgrade_to_full(); ShenandoahFullGC full_gc; full_gc.op_full(GCCause::_shenandoah_upgrade_to_full_gc); } void ShenandoahDegenGC::op_degenerated_futile() { - ShenandoahHeap::heap()->shenandoah_policy()->record_degenerated_upgrade_to_full(); + upgrade_to_full(); ShenandoahFullGC full_gc; full_gc.op_full(GCCause::_shenandoah_upgrade_to_full_gc); } @@ -361,3 +360,13 @@ const char* ShenandoahDegenGC::degen_event_message(ShenandoahDegenPoint point) c return "ERROR"; } } + +void ShenandoahDegenGC::upgrade_to_full() { + log_info(gc)("Degenerate GC upgrading to Full GC"); + ShenandoahHeap::heap()->shenandoah_policy()->record_degenerated_upgrade_to_full(); + _upgraded_to_full = true; +} + +bool ShenandoahDegenGC::upgraded_to_full() { + return _upgraded_to_full; +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.hpp index 80371b620650a..e85c607e2235b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.hpp @@ -35,10 +35,12 @@ class ShenandoahDegenGC : public ShenandoahGC { private: const ShenandoahDegenPoint _degen_point; ShenandoahGeneration* _generation; + bool _upgraded_to_full; public: ShenandoahDegenGC(ShenandoahDegenPoint degen_point, ShenandoahGeneration* generation); bool collect(GCCause::Cause cause); + bool upgraded_to_full(); private: void vmop_degenerated(); @@ -61,6 +63,7 @@ class ShenandoahDegenGC : public ShenandoahGC { void op_degenerated_fail(); const char* degen_event_message(ShenandoahDegenPoint point) const; + void upgrade_to_full(); }; #endif // SHARE_GC_SHENANDOAH_SHENANDOAHDEGENERATEDGC_HPP From 1c2ee14e7edab901d96419bae491d8bb574e0ce6 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Thu, 16 Sep 2021 20:44:27 +0000 Subject: [PATCH 066/254] Reset task queue stats before rset scan Reviewed-by: kdnilsen --- src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp | 5 +++++ src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp | 2 -- src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp | 5 +++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index f3de4fb9cccf7..c5753d40d6b9b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -114,6 +114,11 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) { { ShenandoahBreakpointMarkScope breakpoint_mark_scope; + // Reset task queue stats here, rather than in mark_concurrent_roots + // because remembered set scan will `push` oops into the queues and + // resetting after this happens will lose those counts. + TASKQUEUE_STATS_ONLY(_mark.task_queues()->reset_taskqueue_stats()); + // Concurrent remembered set scanning if (_generation->generation_mode() == YOUNG) { _generation->scan_remembered_set(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp index e096d4549a3f6..ddc280e15867e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp @@ -213,8 +213,6 @@ void ShenandoahConcurrentMark::mark_concurrent_roots() { ShenandoahHeap* const heap = ShenandoahHeap::heap(); assert(!heap->has_forwarded_objects(), "Not expected"); - TASKQUEUE_STATS_ONLY(task_queues()->reset_taskqueue_stats()); - WorkGang* workers = heap->workers(); ShenandoahReferenceProcessor* rp = _generation->ref_processor(); _generation->reserve_task_queues(workers->active_workers()); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp index fd56df72f2a76..8a0ebb16d0816 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp @@ -53,10 +53,11 @@ class outputStream; f(init_mark_gross, "Pause Init Mark (G)") \ f(init_mark, "Pause Init Mark (N)") \ f(init_manage_tlabs, " Manage TLABs") \ - f(init_scan_rset, " Scan Remembered Set") \ - SHENANDOAH_PAR_PHASE_DO(init_scan_rset_, " RS: ", f) \ f(init_update_region_states, " Update Region States") \ \ + f(init_scan_rset, "Concurrent Scan Remembered Set") \ + SHENANDOAH_PAR_PHASE_DO(init_scan_rset_, " RS: ", f) \ + \ f(conc_mark_roots, "Concurrent Mark Roots ") \ SHENANDOAH_PAR_PHASE_DO(conc_mark_roots, " CMR: ", f) \ f(conc_mark, "Concurrent Marking") \ From fd2754268dc0f00d6b46ae84c4de1fa9b957dd17 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Fri, 17 Sep 2021 20:49:19 +0000 Subject: [PATCH 067/254] Fix loop error when reconstructing remembered set for humongous objects Reviewed-by: rkennke --- src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp | 2 +- src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index e018213d8d802..07d0a1be26900 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -98,7 +98,7 @@ class ShenandoahReconstructRememberedSetTask : public AbstractGangTask { scanner->reset_object_range(humongous_region->bottom(), humongous_region->end()); region_index++; humongous_region = heap->get_region(region_index); - } while (humongous_region->bottom() < end_object); + } while (humongous_region->top() < end_object); // Then register the humongous object and DIRTY relevant remembered set cards scanner->register_object_wo_lock(obj_addr); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index 8d0c00278dbcd..c109ca85b656c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -864,7 +864,7 @@ void ShenandoahHeapRegion::set_affiliation(ShenandoahRegionAffiliation new_affil log_debug(gc)("Setting affiliation of Region " SIZE_FORMAT " from %s to %s, top: " PTR_FORMAT ", TAMS: " PTR_FORMAT ", watermark: " PTR_FORMAT ", top_bitmap: " PTR_FORMAT "\n", index(), affiliation_name(_affiliation), affiliation_name(new_affiliation), - p2i(top()), p2i(ctx->top_at_mark_start(this)), p2i(this->get_update_watermark()), p2i(ctx->top_bitmap(this))); + p2i(top()), p2i(ctx->top_at_mark_start(this)), p2i(_update_watermark), p2i(ctx->top_bitmap(this))); } #ifdef ASSERT From 52973cc9b2ef8e5768fbd6fd24604978240a144a Mon Sep 17 00:00:00 2001 From: William Kemper Date: Mon, 20 Sep 2021 16:20:51 +0000 Subject: [PATCH 068/254] Track promotion rate into old generation Reviewed-by: rkennke, kdnilsen --- .../shenandoahAdaptiveHeuristics.cpp | 5 +- .../shenandoahAdaptiveOldHeuristics.cpp | 3 + .../share/gc/shenandoah/shenandoahFreeSet.cpp | 5 +- .../gc/shenandoah/shenandoahGeneration.cpp | 12 ++- .../gc/shenandoah/shenandoahGeneration.hpp | 7 +- .../share/gc/shenandoah/shenandoahHeap.cpp | 75 ++++++++++--------- .../share/gc/shenandoah/shenandoahHeap.hpp | 5 +- .../gc/shenandoah/shenandoahHeap.inline.hpp | 36 ++------- 8 files changed, 77 insertions(+), 71 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp index dba883fa74ace..d1750883abc23 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp @@ -226,7 +226,7 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { return true; } - // Check if are need to learn a bit about the application + // Check if we need to learn a bit about the application const size_t max_learn = ShenandoahLearningSteps; if (_gc_times_learned < max_learn) { size_t init_threshold = capacity / 100 * ShenandoahInitFreeThreshold; @@ -252,6 +252,9 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { double avg_cycle_time = _gc_time_history->davg() + (_margin_of_error_sd * _gc_time_history->dsd()); double avg_alloc_rate = _allocation_rate.upper_bound(_margin_of_error_sd); + log_debug(gc)("%s: average GC time: %.2f ms, allocation rate: %.0f %s/s", + _generation->name(), avg_cycle_time * 1000, byte_size_in_proper_unit(avg_alloc_rate), proper_unit_for_byte_size(avg_alloc_rate)); + if (avg_cycle_time > allocation_headroom / avg_alloc_rate) { log_info(gc)("Trigger (%s): Average GC time (%.2f ms) is above the time for average allocation rate (%.0f %sB/s) to deplete free headroom (" SIZE_FORMAT "%s) (margin of error = %.2f)", _generation->name(), avg_cycle_time * 1000, diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveOldHeuristics.cpp index 7f58bb7b72b05..cf0a0782e515f 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveOldHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveOldHeuristics.cpp @@ -260,6 +260,9 @@ bool ShenandoahAdaptiveOldHeuristics::should_start_gc() { double avg_cycle_time = _gc_time_history->davg() + (_margin_of_error_sd * _gc_time_history->dsd()); double avg_alloc_rate = _allocation_rate.upper_bound(_margin_of_error_sd); + log_debug(gc)("%s: average GC time: %.2f ms, allocation rate: %.0f %s/s", + _generation->name(), avg_cycle_time * 1000, byte_size_in_proper_unit(avg_alloc_rate), proper_unit_for_byte_size(avg_alloc_rate)); + if (avg_cycle_time > allocation_headroom / avg_alloc_rate) { log_info(gc)("Trigger (%s): Average GC time (%.2f ms) is above the time for average allocation rate (%.0f %sB/s) to deplete free headroom (" SIZE_FORMAT "%s) (margin of error = %.2f)", _generation->name(), avg_cycle_time * 1000, diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index 745f79f8b64f8..d6695b510302e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -242,6 +242,7 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah size_t waste = r->free(); if (waste > 0) { increase_used(waste); + _heap->generation_for(req.affiliation())->increase_allocated(waste); _heap->notify_mutator_alloc_words(waste >> LogHeapWordSize, true); } } @@ -384,7 +385,9 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req) { if (remainder != 0) { // Record this remainder as allocation waste - _heap->notify_mutator_alloc_words(ShenandoahHeapRegion::region_size_words() - remainder, true); + size_t waste = ShenandoahHeapRegion::region_size_words() - remainder; + _heap->notify_mutator_alloc_words(waste, true); + _heap->generation_for(req.affiliation())->increase_allocated(waste * HeapWordSize); } // Allocated at left/rightmost? Move the bounds appropriately. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index fd202ed18938f..d690b0d4ba0d1 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -117,7 +117,15 @@ ShenandoahHeuristics* ShenandoahGeneration::initialize_heuristics(ShenandoahMode } size_t ShenandoahGeneration::bytes_allocated_since_gc_start() { - return ShenandoahHeap::heap()->bytes_allocated_since_gc_start(); + return Atomic::load(&_bytes_allocated_since_gc_start);; +} + +void ShenandoahGeneration::reset_bytes_allocated_since_gc_start() { + Atomic::store(&_bytes_allocated_since_gc_start, (size_t)0); +} + +void ShenandoahGeneration::increase_allocated(size_t bytes) { + Atomic::add(&_bytes_allocated_since_gc_start, bytes, memory_order_relaxed); } void ShenandoahGeneration::log_status() const { @@ -273,7 +281,7 @@ ShenandoahGeneration::ShenandoahGeneration(GenerationMode generation_mode, _generation_mode(generation_mode), _task_queues(new ShenandoahObjToScanQueueSet(max_workers)), _ref_processor(new ShenandoahReferenceProcessor(MAX2(max_workers, 1U))), - _affiliated_region_count(0), _used(0), + _affiliated_region_count(0), _used(0), _bytes_allocated_since_gc_start(0), _max_capacity(max_capacity), _soft_max_capacity(soft_max_capacity) { _is_marking_complete.set(); assert(max_workers > 0, "At least one queue"); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index 367973bbfdf86..32bb4e0bc6926 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -50,6 +50,7 @@ class ShenandoahGeneration : public CHeapObj { // Usage size_t _affiliated_region_count; volatile size_t _used; + volatile size_t _bytes_allocated_since_gc_start; size_t _max_capacity; size_t _soft_max_capacity; @@ -75,12 +76,14 @@ class ShenandoahGeneration : public CHeapObj { virtual size_t used() const { return _used; } virtual size_t available() const; + size_t bytes_allocated_since_gc_start(); + void reset_bytes_allocated_since_gc_start(); + void increase_allocated(size_t bytes); + void set_soft_max_capacity(size_t soft_max_capacity) { _soft_max_capacity = soft_max_capacity; } - virtual size_t bytes_allocated_since_gc_start(); - void log_status() const; // Used directly by FullGC diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index ac60d6e56b241..14870e3f9744f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -503,7 +503,6 @@ ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) : _initial_size(0), _used(0), _committed(0), - _bytes_allocated_since_gc_start(0), _max_workers(MAX3(ConcGCThreads, ParallelGCThreads, 1U)), _workers(NULL), _safepoint_workers(NULL), @@ -691,16 +690,12 @@ void ShenandoahHeap::decrease_used(size_t bytes) { Atomic::sub(&_used, bytes, memory_order_relaxed); } -void ShenandoahHeap::increase_allocated(size_t bytes) { - Atomic::add(&_bytes_allocated_since_gc_start, bytes, memory_order_relaxed); -} - void ShenandoahHeap::notify_mutator_alloc_words(size_t words, bool waste) { size_t bytes = words * HeapWordSize; if (!waste) { increase_used(bytes); } - increase_allocated(bytes); + if (ShenandoahPacing) { control_thread()->pacing_notify_alloc(words); if (waste) { @@ -807,6 +802,22 @@ void ShenandoahHeap::op_uncommit(double shrink_before, size_t shrink_until) { } } +void ShenandoahHeap::handle_old_evacuation(HeapWord* obj, size_t words, bool promotion) { + // Only register the copy of the object that won the evacuation race. + card_scan()->register_object_wo_lock(obj); + + // Mark the entire range of the evacuated object as dirty. At next remembered set scan, + // we will clear dirty bits that do not hold interesting pointers. It's more efficient to + // do this in batch, in a background GC thread than to try to carefully dirty only cards + // that hold interesting pointers right now. + card_scan()->mark_range_as_dirty(obj, words); + + if (promotion) { + // This evacuation was a promotion, track this as allocation against old gen + old_generation()->increase_allocated(words * HeapWordSize); + } +} + HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) { // New object should fit the GCLAB size size_t min_size = MAX2(size, PLAB::min_size()); @@ -1019,8 +1030,10 @@ HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) { } if (result != NULL) { + ShenandoahGeneration* alloc_generation = generation_for(req.affiliation()); size_t requested = req.size(); size_t actual = req.actual_size(); + size_t actual_bytes = actual * HeapWordSize; assert (req.is_lab_alloc() || (requested == actual), "Only LAB allocations are elastic: %s, requested = " SIZE_FORMAT ", actual = " SIZE_FORMAT, @@ -1028,6 +1041,7 @@ HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) { if (req.is_mutator_alloc()) { notify_mutator_alloc_words(actual, false); + alloc_generation->increase_allocated(actual_bytes); // If we requested more than we were granted, give the rest back to pacer. // This only matters if we are in the same pacing epoch: do not try to unpace @@ -1036,7 +1050,7 @@ HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) { pacer()->unpace_for_alloc(pacer_epoch, requested - actual); } } else { - increase_used(actual*HeapWordSize); + increase_used(actual_bytes); } } @@ -1045,28 +1059,7 @@ HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) { HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req, bool& in_new_region) { ShenandoahHeapLocker locker(lock()); - HeapWord* result = _free_set->allocate(req, in_new_region); - if (result != NULL && req.affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) { - // Register the newly allocated object while we're holding the global lock since there's no synchronization - // built in to the implementation of register_object(). There are potential races when multiple independent - // threads are allocating objects, some of which might span the same card region. For example, consider - // a card table's memory region within which three objects are being allocated by three different threads: - // - // objects being "concurrently" allocated: - // [-----a------][-----b-----][--------------c------------------] - // [---- card table memory range --------------] - // - // Before any objects are allocated, this card's memory range holds no objects. Note that: - // allocation of object a wants to set the has-object, first-start, and last-start attributes of the preceding card region. - // allocation of object b wants to set the has-object, first-start, and last-start attributes of this card region. - // allocation of object c also wants to set the has-object, first-start, and last-start attributes of this card region. - // - // The thread allocating b and the thread allocating c can "race" in various ways, resulting in confusion, such as last-start - // representing object b while first-start represents object c. This is why we need to require all register_object() - // invocations to be "mutually exclusive" with respect to each card's memory range. - ShenandoahHeap::heap()->card_scan()->register_object(result); - } - return result; + return _free_set->allocate(req, in_new_region); } HeapWord* ShenandoahHeap::mem_allocate(size_t size, @@ -2016,12 +2009,13 @@ address ShenandoahHeap::gc_state_addr() { return (address) ShenandoahHeap::heap()->_gc_state.addr_of(); } -size_t ShenandoahHeap::bytes_allocated_since_gc_start() { - return Atomic::load(&_bytes_allocated_since_gc_start); -} - void ShenandoahHeap::reset_bytes_allocated_since_gc_start() { - Atomic::store(&_bytes_allocated_since_gc_start, (size_t)0); + if (mode()->is_generational()) { + young_generation()->reset_bytes_allocated_since_gc_start(); + old_generation()->reset_bytes_allocated_since_gc_start(); + } + + global_generation()->reset_bytes_allocated_since_gc_start(); } void ShenandoahHeap::set_degenerated_gc_in_progress(bool in_progress) { @@ -2740,3 +2734,16 @@ void ShenandoahHeap::verify_rem_set_at_update_ref() { } } } + +ShenandoahGeneration* ShenandoahHeap::generation_for(ShenandoahRegionAffiliation affiliation) const { + if (!mode()->is_generational()) { + return global_generation(); + } else if (affiliation == YOUNG_GENERATION) { + return young_generation(); + } else if (affiliation == OLD_GENERATION) { + return old_generation(); + } + + ShouldNotReachHere(); + return nullptr; +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index cdb9c59a37658..be35802c3a251 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -223,7 +223,6 @@ class ShenandoahHeap : public CollectedHeap { shenandoah_padding(0); volatile size_t _used; volatile size_t _committed; - volatile size_t _bytes_allocated_since_gc_start; shenandoah_padding(1); static size_t young_generation_capacity(size_t total_capacity); @@ -237,9 +236,7 @@ class ShenandoahHeap : public CollectedHeap { void increase_committed(size_t bytes); void decrease_committed(size_t bytes); - void increase_allocated(size_t bytes); - size_t bytes_allocated_since_gc_start(); void reset_bytes_allocated_since_gc_start(); size_t min_capacity() const; @@ -461,6 +458,7 @@ class ShenandoahHeap : public CollectedHeap { ShenandoahYoungGeneration* young_generation() const { return _young_generation; } ShenandoahGeneration* global_generation() const { return _global_generation; } ShenandoahGeneration* old_generation() const { return _old_generation; } + ShenandoahGeneration* generation_for(ShenandoahRegionAffiliation affiliation) const; ShenandoahCollectorPolicy* shenandoah_policy() const { return _shenandoah_policy; } ShenandoahMode* mode() const { return _gc_mode; } @@ -675,6 +673,7 @@ class ShenandoahHeap : public CollectedHeap { ShenandoahEvacOOMHandler _oom_evac_handler; inline oop try_evacuate_object(oop src, Thread* thread, ShenandoahHeapRegion* from_region, ShenandoahRegionAffiliation target_gen); + void handle_old_evacuation(HeapWord* obj, size_t words, bool promotion); public: static address in_cset_fast_test_addr(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp index 58829b5eaaf8e..3b39da5c47389 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp @@ -311,10 +311,6 @@ inline HeapWord* ShenandoahHeap::allocate_from_plab(Thread* thread, size_t size) if (obj == NULL) { obj = allocate_from_plab_slow(thread, size); } - - if (mode()->is_generational() && obj != NULL) { - ShenandoahHeap::heap()->card_scan()->register_object_wo_lock(obj); - } return obj; } @@ -393,7 +389,7 @@ inline oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, Shenandoah if (copy == NULL) { if (target_gen == OLD_GENERATION && from_region->affiliation() == YOUNG_GENERATION) { - // Indicate that a promotion attempt failed. + // TODO: Inform old generation heuristic of promotion failure return NULL; } @@ -408,35 +404,19 @@ inline oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, Shenandoah Copy::aligned_disjoint_words(cast_from_oop(p), copy, size); oop copy_val = cast_to_oop(copy); - if (target_gen == YOUNG_GENERATION) { - // Increment the age in young copies, absorbing region age. - // (Only retired regions will have more than zero age to pass along.) - - ShenandoahHeap::increase_object_age(copy_val, from_region->age() + 1); - - // Note that p may have been forwarded by another thread, - // anywhere between here and the check above for forwarding. - // In that case try_update_forwardee() below will not be successful - // and the increment we just performed will simply be forgotten, - // but it will have succeeded in said other thread. - } // Try to install the new forwarding pointer. oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val); if (result == copy_val) { + // Successfully evacuated. Our copy is now the public one! if (target_gen == OLD_GENERATION) { - if (alloc_from_lab) { - card_scan()->register_object_wo_lock(copy); - } - // else, allocate_memory_under_lock() has already registered the object - - // Mark the entire range of the evacuated object as dirty. At next remembered set scan, - // we will clear dirty bits that do not hold interesting pointers. It's more efficient to - // do this in batch, in a background GC thread than to try to carefully dirty only cards - // that hold interesting pointers right now. - card_scan()->mark_range_as_dirty(copy, size); + handle_old_evacuation(copy, size, from_region->is_young()); + } else if (target_gen == YOUNG_GENERATION) { + ShenandoahHeap::increase_object_age(copy_val, from_region->age() + 1); + } else { + ShouldNotReachHere(); } - // Successfully evacuated. Our copy is now the public one! + shenandoah_assert_correct(NULL, copy_val); return copy_val; } else { From f9e068dce2fecf6a0705a4685ea2f336e65ea2a5 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Wed, 22 Sep 2021 17:16:08 +0000 Subject: [PATCH 069/254] Fix failing jtreg tests Reviewed-by: kdnilsen --- .../share/gc/shenandoah/shenandoahControlThread.cpp | 7 +++---- .../generational/TestCLIModeGenerational.java | 2 +- .../shenandoah/generational/TestConcurrentEvac.java | 12 ++---------- .../generational/TestSimpleGenerational.java | 7 +------ 4 files changed, 7 insertions(+), 21 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index e2f5373f5e1ce..bd0583bcc9b63 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -36,6 +36,7 @@ #include "gc/shenandoah/shenandoahMark.inline.hpp" #include "gc/shenandoah/shenandoahMonitoringSupport.hpp" #include "gc/shenandoah/shenandoahOopClosures.inline.hpp" +#include "gc/shenandoah/shenandoahOldGC.hpp" #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp" #include "gc/shenandoah/shenandoahUtils.hpp" #include "gc/shenandoah/shenandoahVMOperations.hpp" @@ -47,7 +48,6 @@ #include "memory/metaspaceStats.hpp" #include "memory/universe.hpp" #include "runtime/atomic.hpp" -#include "shenandoahOldGC.hpp" ShenandoahControlThread::ShenandoahControlThread() : ConcurrentGCThread(), @@ -230,6 +230,8 @@ void ShenandoahControlThread::run_service() { assert (!gc_requested || cause != GCCause::_last_gc_cause, "GC cause should be set"); if (gc_requested) { + // GC is starting, bump the internal ID + update_gc_id(); heap->reset_bytes_allocated_since_gc_start(); @@ -273,9 +275,6 @@ void ShenandoahControlThread::run_service() { } } - // GC is finished, bump the internal ID before notifying waiters. - update_gc_id(); - // If this was the requested GC cycle, notify waiters about it if (explicit_gc_requested || implicit_gc_requested) { notify_gc_waiters(); diff --git a/test/hotspot/jtreg/gc/shenandoah/generational/TestCLIModeGenerational.java b/test/hotspot/jtreg/gc/shenandoah/generational/TestCLIModeGenerational.java index beff1fb3113b3..269b236a8dbd5 100644 --- a/test/hotspot/jtreg/gc/shenandoah/generational/TestCLIModeGenerational.java +++ b/test/hotspot/jtreg/gc/shenandoah/generational/TestCLIModeGenerational.java @@ -31,7 +31,7 @@ * @summary Test argument processing for -XX:+ShenandoahGCMode=generational. * @library /testlibrary /test/lib / * @build sun.hotspot.WhiteBox - * @run driver ClassFileInstaller sun.hotspot.WhiteBox + * @run driver jdk.test.lib.helpers.ClassFileInstaller sun.hotspot.WhiteBox * @run main/othervm -Xbootclasspath/a:. * -XX:+IgnoreUnrecognizedVMOptions * -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI diff --git a/test/hotspot/jtreg/gc/shenandoah/generational/TestConcurrentEvac.java b/test/hotspot/jtreg/gc/shenandoah/generational/TestConcurrentEvac.java index 4ffa2394c9859..56fc01d87f1e5 100644 --- a/test/hotspot/jtreg/gc/shenandoah/generational/TestConcurrentEvac.java +++ b/test/hotspot/jtreg/gc/shenandoah/generational/TestConcurrentEvac.java @@ -28,15 +28,7 @@ import java.util.Random; import java.util.HashMap; -/* WARNING! - * As of the date on which this test was added into the jtreg suite, heuristic - * of old-gen GC passes is very simplistic. A further shortcoming of the - * Generational Shenandoah as of introduction of this test is that it does - * not currently support full GC. If garbage collection falls behind mutator - * allocations, a full GC will be triggered and Generational Shenandoah will - * abort itself with an assertion error. Both of these limitations will be - * addressed in future releases of Generational Shenandoah. - * +/* * To avoid the risk of false regressions identified by this test, the heap * size is set artificially high. Though this test is known to run reliably * in 66 MB heap, the heap size for this test run is currently set to 256 MB. @@ -48,7 +40,7 @@ * @summary Confirm that card marking and remembered set scanning do not crash. * @library /testlibrary /test/lib / * @build sun.hotspot.WhiteBox - * @run driver ClassFileInstaller sun.hotspot.WhiteBox + * @run driver jdk.test.lib.helpers.ClassFileInstaller sun.hotspot.WhiteBox * @run main/othervm -Xbootclasspath/a:. * -Xms256m -Xmx256m * -XX:+IgnoreUnrecognizedVMOptions diff --git a/test/hotspot/jtreg/gc/shenandoah/generational/TestSimpleGenerational.java b/test/hotspot/jtreg/gc/shenandoah/generational/TestSimpleGenerational.java index c25ac586184fa..53f61fd4361bf 100644 --- a/test/hotspot/jtreg/gc/shenandoah/generational/TestSimpleGenerational.java +++ b/test/hotspot/jtreg/gc/shenandoah/generational/TestSimpleGenerational.java @@ -33,18 +33,13 @@ * @summary Confirm that card marking and remembered set scanning do not crash. * @library /testlibrary /test/lib / * @build sun.hotspot.WhiteBox - * @run driver ClassFileInstaller sun.hotspot.WhiteBox + * @run driver jdk.test.lib.helpers.ClassFileInstaller sun.hotspot.WhiteBox * @run main/othervm -Xbootclasspath/a:. * -XX:+IgnoreUnrecognizedVMOptions * -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational * gc.shenandoah.generational.TestSimpleGenerational */ - -/* This used to be part of the run command, but caused problems. - * -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI - */ - public class TestSimpleGenerational { private static WhiteBox wb = WhiteBox.getWhiteBox(); static private final int SeedForRandom = 46; From ea8469a118d21e41ecbaca17478bd2e7617dbeed Mon Sep 17 00:00:00 2001 From: William Kemper Date: Wed, 29 Sep 2021 18:19:59 +0000 Subject: [PATCH 070/254] Also register filler object for retired gclabs Reviewed-by: kdnilsen, rkennke --- .../share/gc/shenandoah/shenandoahHeap.cpp | 26 +++++++++++-------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 14870e3f9744f..e402d7a7f82d3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -921,16 +921,20 @@ HeapWord* ShenandoahHeap::allocate_from_plab_slow(Thread* thread, size_t size) { } void ShenandoahHeap::retire_plab(PLAB* plab) { - size_t waste = plab->waste(); - HeapWord* top = plab->top(); - plab->retire(); - if (top != NULL && plab->waste() > waste) { - // If retiring the plab created a filler object, then we - // need to register it with our card scanner so it can - // safely walk the region backing the plab. - log_debug(gc)("retire_plab() is registering remnant of size " SIZE_FORMAT " at " PTR_FORMAT, - plab->waste() - waste, p2i(top)); - card_scan()->register_object_wo_lock(top); + if (!mode()->is_generational()) { + plab->retire(); + } else { + size_t waste = plab->waste(); + HeapWord* top = plab->top(); + plab->retire(); + if (top != NULL && plab->waste() > waste) { + // If retiring the plab created a filler object, then we + // need to register it with our card scanner so it can + // safely walk the region backing the plab. + log_debug(gc)("retire_plab() is registering remnant of size " SIZE_FORMAT " at " PTR_FORMAT, + plab->waste() - waste, p2i(top)); + card_scan()->register_object_wo_lock(top); + } } } @@ -1241,7 +1245,7 @@ class ShenandoahRetireGCLABClosure : public ThreadClosure { void do_thread(Thread* thread) { PLAB* gclab = ShenandoahThreadLocalData::gclab(thread); assert(gclab != NULL, "GCLAB should be initialized for %s", thread->name()); - gclab->retire(); + ShenandoahHeap::heap()->retire_plab(gclab); if (_resize && ShenandoahThreadLocalData::gclab_size(thread) > 0) { ShenandoahThreadLocalData::set_gclab_size(thread, 0); } From 7fe11a23d3bdde1d7fb511c5251a0413f61667a9 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Thu, 30 Sep 2021 15:45:50 +0000 Subject: [PATCH 071/254] Only register retired plab waste in old regions Reviewed-by: kdnilsen --- src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index e402d7a7f82d3..f55d9346869c3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -927,7 +927,7 @@ void ShenandoahHeap::retire_plab(PLAB* plab) { size_t waste = plab->waste(); HeapWord* top = plab->top(); plab->retire(); - if (top != NULL && plab->waste() > waste) { + if (top != NULL && plab->waste() > waste && is_in_old(top)) { // If retiring the plab created a filler object, then we // need to register it with our card scanner so it can // safely walk the region backing the plab. From 85592d91833dea6361f063c31c41e17affeb16a2 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Mon, 4 Oct 2021 20:30:16 +0000 Subject: [PATCH 072/254] Full region promotion Reviewed-by: wkemper --- .../heuristics/shenandoahHeuristics.cpp | 7 + .../gc/shenandoah/shenandoahCollectionSet.cpp | 5 +- .../share/gc/shenandoah/shenandoahFullGC.cpp | 10 +- .../share/gc/shenandoah/shenandoahHeap.cpp | 80 ++++++++- .../gc/shenandoah/shenandoahHeapRegion.cpp | 169 ++++-------------- .../gc/shenandoah/shenandoahHeapRegion.hpp | 12 +- .../share/gc/shenandoah/shenandoahMark.cpp | 1 - .../gc/shenandoah/shenandoahPhaseTimings.hpp | 1 - .../shenandoah/shenandoahScanRemembered.cpp | 1 + .../shenandoahScanRemembered.inline.hpp | 17 +- .../shenandoah/shenandoahYoungGeneration.cpp | 36 ---- .../shenandoah/shenandoahYoungGeneration.hpp | 1 - .../gc/shenandoah/shenandoah_globals.hpp | 20 +++ 13 files changed, 163 insertions(+), 197 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp index 54bf526621552..2ebf2ce912d97 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp @@ -124,6 +124,13 @@ bool ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec // This is our candidate for later consideration. candidates[cand_idx]._region = region; + if (heap->mode()->is_generational() && (region->age() >= InitialTenuringThreshold)) { + // Bias selection of regions that have reached tenure age + for (int i = region->age() - InitialTenuringThreshold; i >= 0; i--) { + // Avoid floating-point math with integer multiply and shift. + garbage = (garbage * ShenandoahTenuredRegionUsageBias) >> ShenandoahTenuredRegionUsageBiasLogBase2; + } + } candidates[cand_idx]._garbage = garbage; cand_idx++; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp index 24a0e6b315220..fe5250944081d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp @@ -84,12 +84,13 @@ void ShenandoahCollectionSet::add_region(ShenandoahHeapRegion* r) { assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint"); assert(Thread::current()->is_VM_thread(), "Must be VMThread"); assert(!is_in(r), "Already in collection set"); + assert(!r->is_humongous(), "Only add regular regions to the collection set"); + _cset_map[r->index()] = 1; _region_count++; + _has_old_regions |= r->is_old(); _garbage += r->garbage(); _used += r->used(); - _has_old_regions |= r->is_old(); - // Update the region status too. State transition would be checked internally. r->make_cset(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index 07d0a1be26900..d1b98be783e71 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -90,15 +90,17 @@ class ShenandoahReconstructRememberedSetTask : public AbstractGangTask { size_t size = obj->size(); HeapWord* end_object = r->bottom() + size; - // First, clear the remembered set - scanner->reset_remset(r->bottom(), size); + // First, clear the remembered set for all spanned humongous regions + size_t num_regions = (size + ShenandoahHeapRegion::region_size_words() - 1) / ShenandoahHeapRegion::region_size_words(); + size_t region_span = num_regions * ShenandoahHeapRegion::region_size_words(); + scanner->reset_remset(r->bottom(), region_span); size_t region_index = r->index(); ShenandoahHeapRegion* humongous_region = heap->get_region(region_index); - do { + while (num_regions-- != 0) { scanner->reset_object_range(humongous_region->bottom(), humongous_region->end()); region_index++; humongous_region = heap->get_region(region_index); - } while (humongous_region->top() < end_object); + } // Then register the humongous object and DIRTY relevant remembered set cards scanner->register_object_wo_lock(obj_addr); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index f55d9346869c3..4bdce019446b9 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -1158,11 +1158,72 @@ class ShenandoahEvacuationTask : public AbstractGangTask { ShenandoahHeapRegion* r; while ((r =_cs->claim_next()) != NULL) { assert(r->has_live(), "Region " SIZE_FORMAT " should have been reclaimed early", r->index()); + _sh->marked_object_iterate(r, &cl); if (ShenandoahPacing) { _sh->pacer()->report_evac(r->used() >> LogHeapWordSize); } + if (_sh->check_cancelled_gc_and_yield(_concurrent)) { + break; + } + } + } +}; + +// Unlike ShenandoahEvacuationTask, this iterates over all regions rather than just the collection set. +// This is needed in order to promote humongous start regions if age() >= tenure threshold. +class ShenandoahGenerationalEvacuationTask : public AbstractGangTask { +private: + ShenandoahHeap* const _sh; + ShenandoahRegionIterator *_regions; + bool _concurrent; +public: + ShenandoahGenerationalEvacuationTask(ShenandoahHeap* sh, + ShenandoahRegionIterator* iterator, + bool concurrent) : + AbstractGangTask("Shenandoah Evacuation"), + _sh(sh), + _regions(iterator), + _concurrent(concurrent) + {} + + void work(uint worker_id) { + if (_concurrent) { + ShenandoahConcurrentWorkerSession worker_session(worker_id); + ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers); + ShenandoahEvacOOMScope oom_evac_scope; + do_work(); + } else { + ShenandoahParallelWorkerSession worker_session(worker_id); + ShenandoahEvacOOMScope oom_evac_scope; + do_work(); + } + } + +private: + void do_work() { + ShenandoahConcurrentEvacuateRegionObjectClosure cl(_sh); + ShenandoahHeapRegion* r; + while ((r = _regions->next()) != nullptr) { + log_debug(gc)("GenerationalEvacuationTask do_work(), looking at %s region " SIZE_FORMAT ", (age: %d) [%s, %s]", + r->is_old()? "old": r->is_young()? "young": "free", r->index(), r->age(), + r->is_active()? "active": "inactive", + r->is_humongous()? (r->is_humongous_start()? "humongous_start": "humongous_continuation"): "regular"); + if (r->is_cset()) { + assert(r->has_live(), "Region " SIZE_FORMAT " should have been reclaimed early", r->index()); + _sh->marked_object_iterate(r, &cl); + if (ShenandoahPacing) { + _sh->pacer()->report_evac(r->used() >> LogHeapWordSize); + } + } else if (r->is_young() && r->is_active() && r->is_humongous_start() && (r->age() > InitialTenuringThreshold)) { + // We promote humongous_start regions along with their affiliated continuations during evacuation rather than + // doing this work during a safepoint. We cannot put humongous regions into the collection set because that + // triggers the load-reference barrier (LRB) to copy on reference fetch. + r->promote_humongous(); + } + // else, region is free, or OLD, or not in collection set, or humongous_continuation, + // or is young humongous_start that is too young to be promoted if (_sh->check_cancelled_gc_and_yield(_concurrent)) { break; @@ -1172,8 +1233,14 @@ class ShenandoahEvacuationTask : public AbstractGangTask { }; void ShenandoahHeap::evacuate_collection_set(bool concurrent) { - ShenandoahEvacuationTask task(this, _collection_set, concurrent); - workers()->run_task(&task); + if (ShenandoahHeap::heap()->mode()->is_generational()) { + ShenandoahRegionIterator regions; + ShenandoahGenerationalEvacuationTask task(this, ®ions, concurrent); + workers()->run_task(&task); + } else { + ShenandoahEvacuationTask task(this, _collection_set, concurrent); + workers()->run_task(&task); + } } void ShenandoahHeap::trash_cset_regions() { @@ -2186,6 +2253,8 @@ class ShenandoahUpdateHeapRefsTask : public AbstractGangTask { while (r != NULL) { HeapWord* update_watermark = r->get_update_watermark(); assert (update_watermark >= r->bottom(), "sanity"); + + log_debug(gc)("ShenandoahUpdateHeapRefsTask::do_work(%u) looking at region " SIZE_FORMAT, worker_id, r->index()); if (r->is_active() && !r->is_cset()) { if (!_heap->mode()->is_generational() || (r->affiliation() == ShenandoahRegionAffiliation::YOUNG_GENERATION)) { _heap->marked_object_oop_iterate(r, &cl, update_watermark); @@ -2361,13 +2430,6 @@ void ShenandoahHeap::rebuild_free_set(bool concurrent) { ShenandoahHeapLocker locker(lock()); _free_set->rebuild(); } - - // HEY! this code and rebuild free set used to be in op_final_updaterefs - if (mode()->is_generational() && is_gc_generation_young() && ShenandoahPromoteTenuredRegions) { - ShenandoahGCPhase phase(ShenandoahPhaseTimings::final_update_refs_promote_tenured_regions); - ShenandoahHeapLocker locker(lock()); - young_generation()->promote_tenured_regions(); - } } void ShenandoahHeap::print_extended_on(outputStream *st) const { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index c109ca85b656c..f0447e02aa653 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -511,55 +511,6 @@ void ShenandoahHeapRegion::global_oop_iterate_objects_and_fill_dead(OopIterateCl } } -// This function does not set card dirty bits. The decision of which cards to dirty is best -// made in the caller's context. -void ShenandoahHeapRegion::fill_dead_and_register_for_promotion() { - ShenandoahHeap* heap = ShenandoahHeap::heap(); - ShenandoahMarkingContext* marking_context = heap->marking_context(); - HeapWord* obj_addr = bottom(); - RememberedScanner* rem_set_scanner = heap->card_scan(); - // Objects allocated above TAMS are not marked, but are considered live for purposes of current GC efforts. - HeapWord* t = marking_context->top_at_mark_start(this); - - assert(!is_humongous(), "no humongous region here"); - assert(heap->active_generation()->is_mark_complete(), "sanity"); - - // end() might be overkill as end of range, but top() may not align with card boundary. - rem_set_scanner->reset_object_range(bottom(), end()); - while (obj_addr < t) { - oop obj = cast_to_oop(obj_addr); - if (marking_context->is_marked(obj)) { - assert(obj->klass() != NULL, "klass should not be NULL"); - // when promoting an entire region, we have to register the marked objects as well - rem_set_scanner->register_object_wo_lock(obj_addr); - obj_addr += obj->size(); - } else { - // Object is not marked. Coalesce and fill dead object with dead neighbors. - HeapWord* next_marked_obj = marking_context->get_next_marked_addr(obj_addr, t); - assert(next_marked_obj <= t, "next marked object cannot exceed top"); - size_t fill_size = next_marked_obj - obj_addr; - assert(fill_size >= (size_t) oopDesc::header_size(), - "fill size " SIZE_FORMAT " for obj @ " PTR_FORMAT ", next_marked: " PTR_FORMAT ", TAMS: " PTR_FORMAT " is too small", - fill_size, p2i(obj_addr), p2i(next_marked_obj), p2i(t)); - ShenandoahHeap::fill_with_object(obj_addr, fill_size); - rem_set_scanner->register_object_wo_lock(obj_addr); - obj_addr = next_marked_obj; - } - } - - // Any object above TAMS and below top() is considered live. - t = top(); - while (obj_addr < t) { - oop obj = cast_to_oop(obj_addr); - assert(obj->klass() != NULL, "klass should not be NULL"); - // when promoting an entire region, we have to register the marked objects as well - rem_set_scanner->register_object_wo_lock(obj_addr); - obj_addr += obj->size(); - } - - // Remembered set scanning stops at top() so no need to fill beyond it. -} - void ShenandoahHeapRegion::oop_iterate_humongous(OopIterateClosure* blk) { assert(is_humongous(), "only humongous region here"); // Find head. @@ -916,94 +867,52 @@ void ShenandoahHeapRegion::set_affiliation(ShenandoahRegionAffiliation new_affil _affiliation = new_affiliation; } -size_t ShenandoahHeapRegion::promote(bool promoting_all) { - // TODO: Not sure why region promotion must be performed at safepoint. Reconsider this requirement. - assert(SafepointSynchronize::is_at_safepoint(), "must be at a safepoint"); - - // Note that region promotion occurs at a safepoint following all evacuation. When a region is promoted, we leave - // its TAMS and update_watermark information as is. - // - // Note that update_watermark represents the state of this region as of the moment at which the most recent evacuation - // began. The value of update_watermark is the same for old regions and young regions, as both participate equally in - // the processes of a mixed evacuation. - // - // The meaning of TAMS is different for young-gen and old-gen regions. For a young-gen region, TAMS represents - // top() at start of most recent young-gen concurrent mark. For an old-gen region, TAMS represents top() at start - // of most recent old-gen concurrent mark(). In the case that a young-gen heap region is promoted into old-gen, - // we can preserve its TAMS information with the following understandings: - // 1. The most recent young-GC concurrent mark phase began at the same time or after the most recent old-GC - // concurrent mark phase. - // 2. After the region is promoted, it is still the case that any object within the region that is beneath TAMS - // and is considered alive for the current old GC pass will be "marked" within the current marking context, and - // any object within the region that is above TAMS will be considered alive for the current old GC pass. Objects - // that were dead at promotion time will all reside below TAMS and will be unmarked. +size_t ShenandoahHeapRegion::promote_humongous() { ShenandoahHeap* heap = ShenandoahHeap::heap(); ShenandoahMarkingContext* marking_context = heap->marking_context(); assert(heap->active_generation()->is_mark_complete(), "sanity"); - assert(affiliation() == YOUNG_GENERATION, "Only young regions can be promoted"); + assert(is_young(), "Only young regions can be promoted"); + assert(is_humongous_start(), "Should not promote humongous continuation in isolation"); + assert(age() >= InitialTenuringThreshold, "Only promote regions that are sufficiently aged"); ShenandoahGeneration* old_generation = heap->old_generation(); ShenandoahGeneration* young_generation = heap->young_generation(); - if (is_humongous_start()) { - oop obj = cast_to_oop(bottom()); - assert(marking_context->is_marked(obj), "promoted humongous object should be alive"); - - // Since the humongous region holds only one object, no lock is necessary for this register_object() invocation. - heap->card_scan()->register_object_wo_lock(bottom()); - size_t index_limit = index() + ShenandoahHeapRegion::required_regions(obj->size() * HeapWordSize); - - // For this region and each humongous continuation region spanned by this humongous object, change - // affiliation to OLD_GENERATION and adjust the generation-use tallies. The remnant of memory - // in the last humongous region that is not spanned by obj is currently not used. - for (size_t i = index(); i < index_limit; i++) { - ShenandoahHeapRegion* r = heap->get_region(i); - log_debug(gc)("promoting region " SIZE_FORMAT ", from " SIZE_FORMAT " to " SIZE_FORMAT, - r->index(), (size_t) r->bottom(), (size_t) r->top()); - if (r->top() < r->end()) { - ShenandoahHeap::fill_with_object(r->top(), (r->end() - r->top()) / HeapWordSize); - heap->card_scan()->register_object_wo_lock(r->top()); - heap->card_scan()->mark_range_as_clean(top(), r->end() - r->top()); - } - // We mark the entire humongous object's range as dirty after loop terminates, so no need to dirty the range here - r->set_affiliation(OLD_GENERATION); - log_debug(gc)("promoting humongous region " SIZE_FORMAT ", dirtying cards from " SIZE_FORMAT " to " SIZE_FORMAT, - i, (size_t) r->bottom(), (size_t) r->top()); - old_generation->increase_used(r->used()); - young_generation->decrease_used(r->used()); - } - if (promoting_all || obj->is_typeArray()) { - // Primitive arrays don't need to be scanned. Likewise, if we are promoting_all, there's nothing - // left in young-gen, so there can exist no "interesting" pointers. See above TODO question about requiring - // region promotion at safepoint. If we're not at a safepoint, then we can't really "promote all" without - // directing new allocations to old-gen. That's probably not what we want. The whole "promote-all strategy" - // probably needs to be revisited at some future point. - heap->card_scan()->mark_range_as_clean(bottom(), obj->size()); - } else { - heap->card_scan()->mark_range_as_dirty(bottom(), obj->size()); - } - return index_limit - index(); + oop obj = cast_to_oop(bottom()); + assert(marking_context->is_marked(obj), "promoted humongous object should be alive"); + + size_t spanned_regions = ShenandoahHeapRegion::required_regions(obj->size() * HeapWordSize); + size_t index_limit = index() + spanned_regions; + + log_debug(gc)("promoting humongous region " SIZE_FORMAT ", spanning " SIZE_FORMAT, index(), spanned_regions); + + // Since this region may have served previously as OLD, it may hold obsolete object range info. + heap->card_scan()->reset_object_range(bottom(), bottom() + spanned_regions * ShenandoahHeapRegion::region_size_words()); + // Since the humongous region holds only one object, no lock is necessary for this register_object() invocation. + heap->card_scan()->register_object_wo_lock(bottom()); + + // For this region and each humongous continuation region spanned by this humongous object, change + // affiliation to OLD_GENERATION and adjust the generation-use tallies. The remnant of memory + // in the last humongous region that is not spanned by obj is currently not used. + for (size_t i = index(); i < index_limit; i++) { + ShenandoahHeapRegion* r = heap->get_region(i); + log_debug(gc)("promoting humongous region " SIZE_FORMAT ", from " PTR_FORMAT " to " PTR_FORMAT, + r->index(), p2i(r->bottom()), p2i(r->top())); + // We mark the entire humongous object's range as dirty after loop terminates, so no need to dirty the range here + r->set_affiliation(OLD_GENERATION); + old_generation->increase_used(r->used()); + young_generation->decrease_used(r->used()); + } + if (obj->is_typeArray()) { + // Primitive arrays don't need to be scanned. See above TODO question about requiring + // region promotion at safepoint. + log_debug(gc)("Clean cards for promoted humongous object (Region " SIZE_FORMAT ") from " PTR_FORMAT " to " PTR_FORMAT, + index(), p2i(bottom()), p2i(bottom() + obj->size())); + heap->card_scan()->mark_range_as_clean(bottom(), obj->size()); } else { - log_debug(gc)("promoting region " SIZE_FORMAT ", dirtying cards from " SIZE_FORMAT " to " SIZE_FORMAT, - index(), (size_t) bottom(), (size_t) top()); - assert(!is_humongous_continuation(), "should not promote humongous object continuation in isolation"); - - fill_dead_and_register_for_promotion(); - // Rather than scanning entire contents of the promoted region right now to determine which - // cards to mark as dirty, we just mark them all as dirty (unless promoting_all). Later, when we - // scan the remembered set, we will clear cards that are found to not contain live references to - // young memory. Ultimately, this approach is more efficient as it only scans the "dirty" cards - // once and the clean cards once. The alternative approach of scanning all cards now and then - // scanning dirty cards again at next concurrent mark pass scans the clean cards once and the dirty - // cards twice. - if (promoting_all) { - heap->card_scan()->mark_range_as_clean(bottom(), top() - bottom()); - } else { - heap->card_scan()->mark_range_as_dirty(bottom(), top() - bottom()); - } - set_affiliation(OLD_GENERATION); - old_generation->increase_used(used()); - young_generation->decrease_used(used()); - return 1; + log_debug(gc)("Dirty cards for promoted humongous object (Region " SIZE_FORMAT ") from " PTR_FORMAT " to " PTR_FORMAT, + index(), p2i(bottom()), p2i(bottom() + obj->size())); + heap->card_scan()->mark_range_as_dirty(bottom(), obj->size()); } + return index_limit - index(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp index 9b962f7c301a1..584e5e69b8852 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp @@ -419,12 +419,8 @@ class ShenandoahHeapRegion { void increment_age() { if (_age < markWord::max_age) { _age++; } } void reset_age() { _age = 0; } - // Adjusts remembered set information by setting all cards to clean if promoting all, setting - // all cards to dirty otherwise. - // - // Returns the number of regions promoted, which is generally one, but may be greater than 1 if - // this is humongous region with multiple continuations. - size_t promote(bool promoting_all); + // Sets all remembered set cards to dirty. Returns the number of regions spanned by the associated humongous object. + size_t promote_humongous(); private: void do_commit(); @@ -434,10 +430,6 @@ class ShenandoahHeapRegion { // objects, but do not need to register the live objects as they are already registered. void global_oop_iterate_objects_and_fill_dead(OopIterateClosure* cl); - // Process the contents of a region when it is being promoted en masse by registering each marked object, coalescing - // contiguous ranges of unmarked objects into registered dead objects. Do not touch card marks. - void fill_dead_and_register_for_promotion(); - inline void internal_increase_live_data(size_t s); void set_state(RegionState to); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp index c8568a8677e45..ed247cb9cf9fb 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp @@ -184,7 +184,6 @@ void ShenandoahMark::mark_loop_work(T* cl, ShenandoahLiveData* live_data, uint w if (CANCELLABLE && heap->check_cancelled_gc_and_yield()) { return; } - while (satb_mq_set.completed_buffers_num() > 0) { satb_mq_set.apply_closure_to_completed_buffer(&drain_satb); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp index 8a0ebb16d0816..e913228fc4b38 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp @@ -118,7 +118,6 @@ class outputStream; f(final_update_refs_update_region_states, " Update Region States") \ f(final_update_refs_trash_cset, " Trash Collection Set") \ f(final_update_refs_rebuild_freeset, " Rebuild Free Set") \ - f(final_update_refs_promote_tenured_regions, " Promote Tenured Regions") \ \ f(conc_cleanup_complete, "Concurrent Cleanup") \ \ diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp index 38cdddba7c045..3f09cbed0de15 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp @@ -106,6 +106,7 @@ void ShenandoahScanRememberedTask::work(uint worker_id) { _rp->set_mark_closure(worker_id, &cl); ShenandoahHeapRegion* region = _regions->next(); while (region != NULL) { + log_debug(gc)("ShenandoahScanRememberedTask::work(%u), looking at region " SIZE_FORMAT, worker_id, region->index()); if (region->affiliation() == OLD_GENERATION) { scanner->process_region(region, &cl); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp index b921820cfda62..17e818f989392 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp @@ -589,6 +589,9 @@ ShenandoahScanRemembered::process_clusters(size_t first_cluster, process_clusters(first_cluster, count, end_of_range, cl, false); } +// Process all objects starting within count clusters beginning with first_cluster for which the start address is +// less than end_of_range. For any such object, process the complete object, even if its end reaches beyond +// end_of_range. template template inline void @@ -655,7 +658,7 @@ ShenandoahScanRemembered::process_clusters(size_t first_cluster, // Future TODO: // For improved efficiency, we might want to give special handling of obj->is_objArray(). In // particular, in that case, we might want to divide the effort for scanning of a very long object array - // between multiple threads. + // between multiple threads. Also, skip parts of the array that are not marked as dirty. if (obj->is_objArray()) { objArrayOop array = objArrayOop(obj); int len = array->length(); @@ -790,16 +793,23 @@ ShenandoahScanRemembered::process_region(ShenandoahHeapRegion *re end_of_range = region->top(); } + log_debug(gc)("Remembered set scan processing Region " SIZE_FORMAT ", from " PTR_FORMAT " to " PTR_FORMAT ", using %s table", + region->index(), p2i(region->bottom()), p2i(end_of_range), + use_write_table? "read/write (updating)": "read (marking)"); // end_of_range may point to the middle of a cluster because region->top() may be different than region->end(). // We want to assure that our process_clusters() request spans all relevant clusters. Note that each cluster // processed will avoid processing beyond end_of_range. + // Note that any object that starts between start_of_range and end_of_range, including humongous objects, will + // be fully processed by process_clusters, even though the object may reach beyond end_of_range. size_t num_heapwords = end_of_range - start_of_range; unsigned int cluster_size = CardTable::card_size_in_words * ShenandoahCardCluster::CardsPerCluster; size_t num_clusters = (size_t) ((num_heapwords - 1 + cluster_size) / cluster_size); - // Remembered set scanner - process_clusters(start_cluster_no, num_clusters, end_of_range, cl, use_write_table); + if (!region->is_humongous_continuation()) { + // Remembered set scanner + process_clusters(start_cluster_no, num_clusters, end_of_range, cl, use_write_table); + } } template @@ -810,6 +820,7 @@ ShenandoahScanRemembered::cluster_for_addr(HeapWordImpl **addr) { return result; } +// This is used only for debug verification so don't worry about making the scan parallel. template inline void ShenandoahScanRemembered::roots_do(OopIterateClosure* cl) { ShenandoahHeap* heap = ShenandoahHeap::heap(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp index cf27c899acdf9..4b04effef8803 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp @@ -50,42 +50,6 @@ void ShenandoahYoungGeneration::set_concurrent_mark_in_progress(bool in_progress } } -class ShenandoahPromoteTenuredRegionsTask : public AbstractGangTask { -private: - ShenandoahRegionIterator* _regions; -public: - volatile size_t _promoted; - - ShenandoahPromoteTenuredRegionsTask(ShenandoahRegionIterator* regions) : - AbstractGangTask("Shenandoah Promote Tenured Regions"), - _regions(regions), - _promoted(0) { - } - - void work(uint worker_id) { - ShenandoahParallelWorkerSession worker_session(worker_id); - ShenandoahHeapRegion* r = _regions->next(); - while (r != NULL) { - if (r->is_young() && r->age() >= InitialTenuringThreshold && ((r->is_regular() && !r->has_young_lab_flag()) || r->is_humongous_start())) { - // The thread that first encounters a humongous start region promotes the associated humonogous continuations, - // so we do not process humongous continuations directly. Below, we rely on promote() to promote related - // continuation regions when encountering a homongous start. - size_t promoted = r->promote(false); - Atomic::add(&_promoted, promoted); - } - r = _regions->next(); - } - } -}; - -void ShenandoahYoungGeneration::promote_tenured_regions() { - ShenandoahHeap::heap()->set_young_lab_region_flags(); - ShenandoahRegionIterator regions; - ShenandoahPromoteTenuredRegionsTask task(®ions); - ShenandoahHeap::heap()->workers()->run_task(&task); - log_info(gc)("Promoted " SIZE_FORMAT " regions.", task._promoted); -} - bool ShenandoahYoungGeneration::contains(ShenandoahHeapRegion* region) const { // TODO: why not test for equals YOUNG_GENERATION? As written, returns true for regions that are FREE return region->affiliation() != OLD_GENERATION; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp index 70ce349155a4d..5b1664cd089fd 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp @@ -44,7 +44,6 @@ class ShenandoahYoungGeneration : public ShenandoahGeneration { bool contains(ShenandoahHeapRegion* region) const override; bool contains(oop obj) const override; - void promote_tenured_regions(); void set_old_gen_task_queues(ShenandoahObjToScanQueueSet* old_gen_queues) { _old_gen_task_queues = old_gen_queues; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp index 044875e228fe1..80b2851979906 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp @@ -378,6 +378,23 @@ "Testing: use simplified, very inefficient but much less complex" \ " card table scanning.") \ \ + product(uintx, ShenandoahTenuredRegionUsageBias, 192, EXPERIMENTAL, \ + "The collection set is comprised of heap regions that contain " \ + "the greatest amount of garbage. " \ + "For purposes of selecting regions to be included in the " \ + "collection set, regions that have reached the tenure age will " \ + "be treated as if their contained garbage is the actual " \ + "contained garbage multiplied by " \ + "(ShenandoahTenuredRegionUsageBias / 128 (e.g. 150% for the " \ + "default value) as many times as the region's age meets or " \ + "exceeds the tenure age. For example, if tenure age is 7, " \ + "the region age is 9, ShenandoahTenuredRegionUsageBias is " \ + "192, and the region is 12.5% garbage, this region " \ + "will by treated as if its garbage content is " \ + "1/8 * 27/8 = 42.2% when comparing this region to untenured " \ + "regions.") \ + range(128, 256) \ + \ product(bool, ShenandoahPromoteTenuredObjects, true, DIAGNOSTIC, \ "Turn on/off evacuating individual tenured young objects " \ " to the old generation.") \ @@ -391,4 +408,7 @@ " marking in the old generation.") // end of GC_SHENANDOAH_FLAGS +// 2^ShenandoahTenuredRegionUsageBiasLogBase2 is 128 +#define ShenandoahTenuredRegionUsageBiasLogBase2 7 + #endif // SHARE_GC_SHENANDOAH_SHENANDOAH_GLOBALS_HPP From 2493e37ee8120b3f5db9fa9ef6640be207396503 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Mon, 4 Oct 2021 23:20:08 +0000 Subject: [PATCH 073/254] Coalesce and fill unmarked objects during global degenerated cycle Reviewed-by: kdnilsen --- .../gc/shenandoah/shenandoahConcurrentGC.cpp | 18 +----------------- .../gc/shenandoah/shenandoahDegeneratedGC.cpp | 8 ++++++++ .../gc/shenandoah/shenandoahDegeneratedGC.hpp | 1 + .../share/gc/shenandoah/shenandoahHeap.cpp | 18 ++++++++++++++++++ .../share/gc/shenandoah/shenandoahHeap.hpp | 1 + 5 files changed, 29 insertions(+), 17 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index c5753d40d6b9b..c403fe277eb11 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -72,20 +72,6 @@ class ShenandoahBreakpointMarkScope : public StackObj { } }; -class ShenandoahGlobalCoalesceAndFill : public ShenandoahHeapRegionClosure { - public: - virtual void heap_region_do(ShenandoahHeapRegion* region) override { - // old region is not in the collection set and was not immediately trashed - if (region->is_old() && region->is_active() && !region->is_humongous()) { - region->oop_fill_and_coalesce(); - } - } - - virtual bool is_thread_safe() override { - return true; - } -}; - ShenandoahConcurrentGC::ShenandoahConcurrentGC(ShenandoahGeneration* generation, bool do_old_gc_bootstrap) : _mark(generation), _degen_point(ShenandoahDegenPoint::_degenerated_unset), @@ -1068,9 +1054,7 @@ void ShenandoahConcurrentGC::op_cleanup_complete() { } void ShenandoahConcurrentGC::op_global_coalesce_and_fill() { - ShenandoahHeap* const heap = ShenandoahHeap::heap(); - ShenandoahGlobalCoalesceAndFill coalesce; - heap->parallel_heap_region_iterate(&coalesce); + ShenandoahHeap::heap()->coalesce_and_fill_old_regions(); } bool ShenandoahConcurrentGC::check_cancellation_and_abort(ShenandoahDegenPoint point) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp index 08116621696cf..b406969da14d4 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp @@ -134,6 +134,10 @@ void ShenandoahDegenGC::op_degenerated() { op_cleanup_early(); + if (heap->mode()->is_generational() && _generation->generation_mode() == GLOBAL) { + op_global_coalesce_and_fill(); + } + case _degenerated_evac: // If heuristics thinks we should do the cycle, this flag would be set, // and we can do evacuation. Otherwise, it would be the shortcut cycle. @@ -282,6 +286,10 @@ void ShenandoahDegenGC::op_cleanup_early() { ShenandoahHeap::heap()->recycle_trash(); } +void ShenandoahDegenGC::op_global_coalesce_and_fill() { + ShenandoahHeap::heap()->coalesce_and_fill_old_regions(); +} + void ShenandoahDegenGC::op_evacuate() { ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc_stw_evac); ShenandoahHeap::heap()->evacuate_collection_set(false /* concurrent*/); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.hpp index e85c607e2235b..53bf76a4f8c3d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.hpp @@ -52,6 +52,7 @@ class ShenandoahDegenGC : public ShenandoahGC { void op_finish_mark(); void op_prepare_evacuation(); void op_cleanup_early(); + void op_global_coalesce_and_fill(); void op_evacuate(); void op_init_updaterefs(); void op_updaterefs(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 4bdce019446b9..7e909e2499c59 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -943,6 +943,24 @@ void ShenandoahHeap::cancel_mixed_collections() { _old_heuristics->abandon_collection_candidates(); } +void ShenandoahHeap::coalesce_and_fill_old_regions() { + class ShenandoahGlobalCoalesceAndFill : public ShenandoahHeapRegionClosure { + public: + virtual void heap_region_do(ShenandoahHeapRegion* region) override { + // old region is not in the collection set and was not immediately trashed + if (region->is_old() && region->is_active() && !region->is_humongous()) { + region->oop_fill_and_coalesce(); + } + } + + virtual bool is_thread_safe() override { + return true; + } + }; + ShenandoahGlobalCoalesceAndFill coalesce; + parallel_heap_region_iterate(&coalesce); +} + HeapWord* ShenandoahHeap::allocate_new_tlab(size_t min_size, size_t requested_size, size_t* actual_size) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index be35802c3a251..2a2506cf28166 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -707,6 +707,7 @@ class ShenandoahHeap : public CollectedHeap { void mark_card_as_dirty(void* location); void retire_plab(PLAB* plab); void cancel_mixed_collections(); + void coalesce_and_fill_old_regions(); // ---------- Helper functions // From 4d8ce22f209f125a456cc757842f10a459822e60 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Tue, 5 Oct 2021 16:23:06 +0000 Subject: [PATCH 074/254] Decouple generational heuristics Reviewed-by: kdnilsen --- .../shenandoahAdaptiveHeuristics.cpp | 4 +- .../shenandoahAdaptiveOldHeuristics.cpp | 322 ------------------ .../shenandoahAdaptiveOldHeuristics.hpp | 111 ------ .../shenandoahAggressiveOldHeuristics.cpp | 69 ---- .../shenandoahAggressiveOldHeuristics.hpp | 47 --- .../shenandoahCompactOldHeuristics.cpp | 99 ------ .../shenandoahCompactOldHeuristics.hpp | 45 --- .../heuristics/shenandoahHeuristics.cpp | 7 +- .../heuristics/shenandoahHeuristics.hpp | 6 + .../heuristics/shenandoahOldHeuristics.cpp | 148 ++++++-- .../heuristics/shenandoahOldHeuristics.hpp | 42 ++- .../shenandoahPassiveOldHeuristics.cpp | 75 ---- .../shenandoahPassiveOldHeuristics.hpp | 50 --- .../shenandoahStaticOldHeuristics.cpp | 74 ---- .../shenandoahStaticOldHeuristics.hpp | 47 --- .../gc/shenandoah/mode/shenandoahMode.cpp | 21 -- .../gc/shenandoah/mode/shenandoahMode.hpp | 1 - .../shenandoah/mode/shenandoahPassiveMode.cpp | 6 - .../shenandoah/mode/shenandoahPassiveMode.hpp | 1 - .../gc/shenandoah/shenandoahGeneration.cpp | 11 +- .../gc/shenandoah/shenandoahGeneration.hpp | 8 +- .../share/gc/shenandoah/shenandoahHeap.cpp | 16 +- .../share/gc/shenandoah/shenandoahHeap.hpp | 5 +- .../gc/shenandoah/shenandoahOldGeneration.cpp | 24 ++ .../gc/shenandoah/shenandoahOldGeneration.hpp | 2 + .../shenandoah/shenandoahRegulatorThread.cpp | 4 +- .../shenandoah/shenandoahRegulatorThread.hpp | 9 +- .../shenandoahScanRemembered.inline.hpp | 1 - .../gc/shenandoah/shenandoah_globals.hpp | 22 ++ 29 files changed, 244 insertions(+), 1033 deletions(-) delete mode 100644 src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveOldHeuristics.cpp delete mode 100644 src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveOldHeuristics.hpp delete mode 100644 src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveOldHeuristics.cpp delete mode 100644 src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveOldHeuristics.hpp delete mode 100644 src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactOldHeuristics.cpp delete mode 100644 src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactOldHeuristics.hpp delete mode 100644 src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveOldHeuristics.cpp delete mode 100644 src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveOldHeuristics.hpp delete mode 100644 src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticOldHeuristics.cpp delete mode 100644 src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticOldHeuristics.hpp diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp index d1750883abc23..35c5786b3b84b 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp @@ -201,9 +201,9 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { size_t available = _generation->available(); size_t allocated = _generation->bytes_allocated_since_gc_start(); - log_debug(gc)("should_start_gc? available: " SIZE_FORMAT ", soft_max_capacity: " SIZE_FORMAT + log_debug(gc)("should_start_gc (%s)? available: " SIZE_FORMAT ", soft_max_capacity: " SIZE_FORMAT ", max_capacity: " SIZE_FORMAT ", allocated: " SIZE_FORMAT, - available, capacity, max_capacity, allocated); + _generation->name(), available, capacity, max_capacity, allocated); // Make sure the code below treats available without the soft tail. size_t soft_tail = max_capacity - capacity; diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveOldHeuristics.cpp deleted file mode 100644 index cf0a0782e515f..0000000000000 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveOldHeuristics.cpp +++ /dev/null @@ -1,322 +0,0 @@ -/* - * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" - -#include "gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp" -#include "gc/shenandoah/heuristics/shenandoahAdaptiveOldHeuristics.hpp" -#include "gc/shenandoah/shenandoahCollectionSet.hpp" -#include "gc/shenandoah/shenandoahFreeSet.hpp" -#include "gc/shenandoah/shenandoahGeneration.hpp" -#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" -#include "logging/log.hpp" -#include "logging/logTag.hpp" -#include "utilities/quickSort.hpp" - -// These constants are used to adjust the margin of error for the moving -// average of the allocation rate and cycle time. The units are standard -// deviations. -const double ShenandoahAdaptiveOldHeuristics::FULL_PENALTY_SD = 0.2; -const double ShenandoahAdaptiveOldHeuristics::DEGENERATE_PENALTY_SD = 0.1; - -// These are used to decide if we want to make any adjustments at all -// at the end of a successful concurrent cycle. -const double ShenandoahAdaptiveOldHeuristics::LOWEST_EXPECTED_AVAILABLE_AT_END = -0.5; -const double ShenandoahAdaptiveOldHeuristics::HIGHEST_EXPECTED_AVAILABLE_AT_END = 0.5; - -// These values are the confidence interval expressed as standard deviations. -// At the minimum confidence level, there is a 25% chance that the true value of -// the estimate (average cycle time or allocation rate) is not more than -// MINIMUM_CONFIDENCE standard deviations away from our estimate. Similarly, the -// MAXIMUM_CONFIDENCE interval here means there is a one in a thousand chance -// that the true value of our estimate is outside the interval. These are used -// as bounds on the adjustments applied at the outcome of a GC cycle. -const double ShenandoahAdaptiveOldHeuristics::MINIMUM_CONFIDENCE = 0.319; // 25% -const double ShenandoahAdaptiveOldHeuristics::MAXIMUM_CONFIDENCE = 3.291; // 99.9% - -ShenandoahAdaptiveOldHeuristics::ShenandoahAdaptiveOldHeuristics(ShenandoahGeneration* generation) : - ShenandoahOldHeuristics(generation), - _margin_of_error_sd(ShenandoahAdaptiveInitialConfidence), - _spike_threshold_sd(ShenandoahAdaptiveInitialSpikeThreshold), - _last_trigger(OTHER) { } - -ShenandoahAdaptiveOldHeuristics::~ShenandoahAdaptiveOldHeuristics() {} - -void ShenandoahAdaptiveOldHeuristics::choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset, - RegionData* data, size_t size, - size_t actual_free) { - size_t garbage_threshold = ShenandoahHeapRegion::region_size_bytes() * ShenandoahGarbageThreshold / 100; - - // The logic for cset selection in adaptive is as follows: - // - // 1. We cannot get cset larger than available free space. Otherwise we guarantee OOME - // during evacuation, and thus guarantee full GC. In practice, we also want to let - // application to allocate something. This is why we limit CSet to some fraction of - // available space. In non-overloaded heap, max_cset would contain all plausible candidates - // over garbage threshold. - // - // 2. We should not get cset too low so that free threshold would not be met right - // after the cycle. Otherwise we get back-to-back cycles for no reason if heap is - // too fragmented. In non-overloaded non-fragmented heap min_garbage would be around zero. - // - // Therefore, we start by sorting the regions by garbage. Then we unconditionally add the best candidates - // before we meet min_garbage. Then we add all candidates that fit with a garbage threshold before - // we hit max_cset. When max_cset is hit, we terminate the cset selection. Note that in this scheme, - // ShenandoahGarbageThreshold is the soft threshold which would be ignored until min_garbage is hit. - - size_t capacity = _generation->soft_max_capacity(); - size_t max_cset = (size_t)((1.0 * capacity / 100 * ShenandoahEvacReserve) / ShenandoahEvacWaste); - size_t free_target = (capacity / 100 * ShenandoahMinFreeThreshold) + max_cset; - size_t min_garbage = (free_target > actual_free ? (free_target - actual_free) : 0); - - log_info(gc, ergo)("Adaptive CSet Selection. Target Free: " SIZE_FORMAT "%s, Actual Free: " - SIZE_FORMAT "%s, Max CSet: " SIZE_FORMAT "%s, Min Garbage: " SIZE_FORMAT "%s", - byte_size_in_proper_unit(free_target), proper_unit_for_byte_size(free_target), - byte_size_in_proper_unit(actual_free), proper_unit_for_byte_size(actual_free), - byte_size_in_proper_unit(max_cset), proper_unit_for_byte_size(max_cset), - byte_size_in_proper_unit(min_garbage), proper_unit_for_byte_size(min_garbage)); - - - // Better select garbage-first regions - QuickSort::sort(data, (int)size, compare_by_garbage, false); - - size_t cur_cset = 0; - size_t cur_garbage = 0; - - for (size_t idx = 0; idx < size; idx++) { - ShenandoahHeapRegion* r = data[idx]._region; - - size_t new_cset = cur_cset + r->get_live_data_bytes(); - size_t new_garbage = cur_garbage + r->garbage(); - - if (new_cset > max_cset) { - break; - } - - if ((new_garbage < min_garbage) || (r->garbage() > garbage_threshold)) { - cset->add_region(r); - cur_cset = new_cset; - cur_garbage = new_garbage; - } - } -} - -void ShenandoahAdaptiveOldHeuristics::record_cycle_start() { - ShenandoahHeuristics::record_cycle_start(); - _allocation_rate.allocation_counter_reset(); -} - -void ShenandoahAdaptiveOldHeuristics::record_success_concurrent() { - ShenandoahHeuristics::record_success_concurrent(); - - size_t available = ShenandoahHeap::heap()->free_set()->available(); - - _available.add(available); - double z_score = 0.0; - if (_available.sd() > 0) { - z_score = (available - _available.avg()) / _available.sd(); - } - - log_debug(gc, ergo)("Available: " SIZE_FORMAT " %sB, z-score=%.3f. Average available: %.1f %sB +/- %.1f %sB.", - byte_size_in_proper_unit(available), proper_unit_for_byte_size(available), - z_score, - byte_size_in_proper_unit(_available.avg()), proper_unit_for_byte_size(_available.avg()), - byte_size_in_proper_unit(_available.sd()), proper_unit_for_byte_size(_available.sd())); - - // In the case when a concurrent GC cycle completes successfully but with an - // unusually small amount of available memory we will adjust our trigger - // parameters so that they are more likely to initiate a new cycle. - // Conversely, when a GC cycle results in an above average amount of available - // memory, we will adjust the trigger parameters to be less likely to initiate - // a GC cycle. - // - // The z-score we've computed is in no way statistically related to the - // trigger parameters, but it has the nice property that worse z-scores for - // available memory indicate making larger adjustments to the trigger - // parameters. It also results in fewer adjustments as the application - // stabilizes. - // - // In order to avoid making endless and likely unnecessary adjustments to the - // trigger parameters, the change in available memory (with respect to the - // average) at the end of a cycle must be beyond these threshold values. - if (z_score < LOWEST_EXPECTED_AVAILABLE_AT_END || - z_score > HIGHEST_EXPECTED_AVAILABLE_AT_END) { - // The sign is flipped because a negative z-score indicates that the - // available memory at the end of the cycle is below average. Positive - // adjustments make the triggers more sensitive (i.e., more likely to fire). - // The z-score also gives us a measure of just how far below normal. This - // property allows us to adjust the trigger parameters proportionally. - // - // The `100` here is used to attenuate the size of our adjustments. This - // number was chosen empirically. It also means the adjustments at the end of - // a concurrent cycle are an order of magnitude smaller than the adjustments - // made for a degenerated or full GC cycle (which themselves were also - // chosen empirically). - adjust_last_trigger_parameters(z_score / -100); - } -} - -void ShenandoahAdaptiveOldHeuristics::record_success_degenerated() { - ShenandoahHeuristics::record_success_degenerated(); - // Adjust both trigger's parameters in the case of a degenerated GC because - // either of them should have triggered earlier to avoid this case. - adjust_margin_of_error(DEGENERATE_PENALTY_SD); - adjust_spike_threshold(DEGENERATE_PENALTY_SD); -} - -void ShenandoahAdaptiveOldHeuristics::record_success_full() { - ShenandoahHeuristics::record_success_full(); - // Adjust both trigger's parameters in the case of a full GC because - // either of them should have triggered earlier to avoid this case. - adjust_margin_of_error(FULL_PENALTY_SD); - adjust_spike_threshold(FULL_PENALTY_SD); -} - -static double saturate(double value, double min, double max) { - return MAX2(MIN2(value, max), min); -} - -bool ShenandoahAdaptiveOldHeuristics::should_start_gc() { - size_t max_capacity = _generation->max_capacity(); - size_t capacity = _generation->soft_max_capacity(); - size_t available = _generation->available(); - // TODO: Fix implementation of old_generation->bytes_allocated_since_gc_start() to represent bytes promoted since - // start of most recent OLD collection. - - // Note further: available is the difference between soft_capacity and in_use. So soft_tail has already been removed - // from this total. It is redundant to remove it again below. - - size_t allocated = _generation->bytes_allocated_since_gc_start(); - - log_debug(gc)("should_start_old_gc? available: " SIZE_FORMAT ", soft_max_capacity: " SIZE_FORMAT ", max_capacity: " SIZE_FORMAT, - available, capacity, max_capacity); - log_debug(gc)(" allocated: " SIZE_FORMAT, allocated); - - // Make sure the code below treats available without the soft tail. - size_t soft_tail = max_capacity - capacity; - available = (available > soft_tail) ? (available - soft_tail) : 0; - - // Track allocation rate even if we decide to start a cycle for other reasons. - double rate = _allocation_rate.sample(allocated); - _last_trigger = OTHER; - - size_t min_threshold = (capacity * ShenandoahMinFreeThreshold) / 100; - - log_debug(gc)(" available adjusted to: " SIZE_FORMAT ", min_threshold: " SIZE_FORMAT ", ShenandoahMinFreeThreshold: " SIZE_FORMAT, - available, min_threshold, ShenandoahMinFreeThreshold); - - if (available < min_threshold) { - log_info(gc)("Trigger (%s): Free (" SIZE_FORMAT "%s) is below minimum threshold (" SIZE_FORMAT "%s)", - _generation->name(), - byte_size_in_proper_unit(available), proper_unit_for_byte_size(available), - byte_size_in_proper_unit(min_threshold), proper_unit_for_byte_size(min_threshold)); - return true; - } - - // Check if we need to learn a bit about the application - const size_t max_learn = ShenandoahLearningSteps; - if (_gc_times_learned < max_learn) { - size_t init_threshold = capacity / 100 * ShenandoahInitFreeThreshold; - if (available < init_threshold) { - log_info(gc)("Trigger (%s): Learning " SIZE_FORMAT " of " SIZE_FORMAT ". Free (" SIZE_FORMAT "%s) is below initial threshold (" SIZE_FORMAT "%s)", - _generation->name(), _gc_times_learned + 1, max_learn, - byte_size_in_proper_unit(available), proper_unit_for_byte_size(available), - byte_size_in_proper_unit(init_threshold), proper_unit_for_byte_size(init_threshold)); - return true; - } - } - - // Check if allocation headroom is still okay. This also factors in: - // 1. Some space to absorb allocation spikes - // 2. Accumulated penalties from Degenerated and Full GC - size_t allocation_headroom = available; - - size_t spike_headroom = capacity / 100 * ShenandoahAllocSpikeFactor; - size_t penalties = capacity / 100 * _gc_time_penalties; - - allocation_headroom -= MIN2(allocation_headroom, spike_headroom); - allocation_headroom -= MIN2(allocation_headroom, penalties); - - double avg_cycle_time = _gc_time_history->davg() + (_margin_of_error_sd * _gc_time_history->dsd()); - double avg_alloc_rate = _allocation_rate.upper_bound(_margin_of_error_sd); - log_debug(gc)("%s: average GC time: %.2f ms, allocation rate: %.0f %s/s", - _generation->name(), avg_cycle_time * 1000, byte_size_in_proper_unit(avg_alloc_rate), proper_unit_for_byte_size(avg_alloc_rate)); - - if (avg_cycle_time > allocation_headroom / avg_alloc_rate) { - log_info(gc)("Trigger (%s): Average GC time (%.2f ms) is above the time for average allocation rate (%.0f %sB/s) to deplete free headroom (" SIZE_FORMAT "%s) (margin of error = %.2f)", - _generation->name(), avg_cycle_time * 1000, - byte_size_in_proper_unit(avg_alloc_rate), proper_unit_for_byte_size(avg_alloc_rate), - byte_size_in_proper_unit(allocation_headroom), proper_unit_for_byte_size(allocation_headroom), - _margin_of_error_sd); - - log_info(gc, ergo)("Free headroom: " SIZE_FORMAT "%s (free) - " SIZE_FORMAT "%s (spike) - " SIZE_FORMAT "%s (penalties) = " SIZE_FORMAT "%s", - byte_size_in_proper_unit(available), proper_unit_for_byte_size(available), - byte_size_in_proper_unit(spike_headroom), proper_unit_for_byte_size(spike_headroom), - byte_size_in_proper_unit(penalties), proper_unit_for_byte_size(penalties), - byte_size_in_proper_unit(allocation_headroom), proper_unit_for_byte_size(allocation_headroom)); - - _last_trigger = RATE; - return true; - } - - bool is_spiking = _allocation_rate.is_spiking(rate, _spike_threshold_sd); - if (is_spiking && avg_cycle_time > allocation_headroom / rate) { - log_info(gc)("Trigger (%s): Average GC time (%.2f ms) is above the time for instantaneous allocation rate (%.0f %sB/s) to deplete free headroom (" SIZE_FORMAT "%s) (spike threshold = %.2f)", - _generation->name(), avg_cycle_time * 1000, - byte_size_in_proper_unit(rate), proper_unit_for_byte_size(rate), - byte_size_in_proper_unit(allocation_headroom), proper_unit_for_byte_size(allocation_headroom), - _spike_threshold_sd); - _last_trigger = SPIKE; - return true; - } - - return ShenandoahHeuristics::should_start_gc(); -} - -void ShenandoahAdaptiveOldHeuristics::adjust_last_trigger_parameters(double amount) { - switch (_last_trigger) { - case RATE: - adjust_margin_of_error(amount); - break; - case SPIKE: - adjust_spike_threshold(amount); - break; - case OTHER: - // nothing to adjust here. - break; - default: - ShouldNotReachHere(); - } -} - -void ShenandoahAdaptiveOldHeuristics::adjust_margin_of_error(double amount) { - _margin_of_error_sd = saturate(_margin_of_error_sd + amount, MINIMUM_CONFIDENCE, MAXIMUM_CONFIDENCE); - log_debug(gc, ergo)("Margin of error now %.2f", _margin_of_error_sd); -} - -void ShenandoahAdaptiveOldHeuristics::adjust_spike_threshold(double amount) { - _spike_threshold_sd = saturate(_spike_threshold_sd - amount, MINIMUM_CONFIDENCE, MAXIMUM_CONFIDENCE); - log_debug(gc, ergo)("Spike threshold now: %.2f", _spike_threshold_sd); -} - diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveOldHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveOldHeuristics.hpp deleted file mode 100644 index fce1eb8552b49..0000000000000 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveOldHeuristics.hpp +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHADAPTIVEOLDHEURISTICS_HPP -#define SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHADAPTIVEOLDHEURISTICS_HPP - -#include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp" -#include "gc/shenandoah/shenandoahPhaseTimings.hpp" -#include "utilities/numberSeq.hpp" - -class ShenandoahAllocationRate; - -class ShenandoahAdaptiveOldHeuristics : public ShenandoahOldHeuristics { -public: - ShenandoahAdaptiveOldHeuristics(ShenandoahGeneration* generation); - - virtual ~ShenandoahAdaptiveOldHeuristics(); - - virtual void choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset, - RegionData* data, size_t size, - size_t actual_free); - - void record_cycle_start(); - void record_success_concurrent(); - void record_success_degenerated(); - void record_success_full(); - - virtual bool should_start_gc(); - - virtual const char* name() { return "AdaptiveOld"; } - virtual bool is_diagnostic() { return false; } - virtual bool is_experimental() { return false; } - - private: - // These are used to adjust the margin of error and the spike threshold - // in response to GC cycle outcomes. These values are shared, but the - // margin of error and spike threshold trend in opposite directions. - const static double FULL_PENALTY_SD; - const static double DEGENERATE_PENALTY_SD; - - const static double MINIMUM_CONFIDENCE; - const static double MAXIMUM_CONFIDENCE; - - const static double LOWEST_EXPECTED_AVAILABLE_AT_END; - const static double HIGHEST_EXPECTED_AVAILABLE_AT_END; - - friend class ShenandoahAllocationRate; - - // Used to record the last trigger that signaled to start a GC. - // This itself is used to decide whether or not to adjust the margin of - // error for the average cycle time and allocation rate or the allocation - // spike detection threshold. - enum Trigger { - SPIKE, RATE, OTHER - }; - - void adjust_last_trigger_parameters(double amount); - void adjust_margin_of_error(double amount); - void adjust_spike_threshold(double amount); - - ShenandoahAllocationRate _allocation_rate; - - // The margin of error expressed in standard deviations to add to our - // average cycle time and allocation rate. As this value increases we - // tend to over estimate the rate at which mutators will deplete the - // heap. In other words, erring on the side of caution will trigger more - // concurrent GCs. - double _margin_of_error_sd; - - // The allocation spike threshold is expressed in standard deviations. - // If the standard deviation of the most recent sample of the allocation - // rate exceeds this threshold, a GC cycle is started. As this value - // decreases the sensitivity to allocation spikes increases. In other - // words, lowering the spike threshold will tend to increase the number - // of concurrent GCs. - double _spike_threshold_sd; - - // Remember which trigger is responsible for the last GC cycle. When the - // outcome of the cycle is evaluated we will adjust the parameters for the - // corresponding triggers. Note that successful outcomes will raise - // the spike threshold and lower the margin of error. - Trigger _last_trigger; - - // Keep track of the available memory at the end of a GC cycle. This - // establishes what is 'normal' for the application and is used as a - // source of feedback to adjust trigger parameters. - TruncatedSeq _available; -}; - -#endif // SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHADAPTIVEOLDHEURISTICS_HPP diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveOldHeuristics.cpp deleted file mode 100644 index 2ba15a7d410b8..0000000000000 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveOldHeuristics.cpp +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" - -#include "gc/shenandoah/heuristics/shenandoahAggressiveOldHeuristics.hpp" -#include "gc/shenandoah/shenandoahCollectionSet.hpp" -#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" -#include "logging/log.hpp" -#include "logging/logTag.hpp" -#include "runtime/os.hpp" - -ShenandoahAggressiveOldHeuristics::ShenandoahAggressiveOldHeuristics(ShenandoahGeneration* generation) : ShenandoahOldHeuristics(generation) { - // Do not shortcut evacuation - SHENANDOAH_ERGO_OVERRIDE_DEFAULT(ShenandoahImmediateThreshold, 100); - - // Aggressive evacuates everything, so it needs as much evac space as it can get - SHENANDOAH_ERGO_ENABLE_FLAG(ShenandoahEvacReserveOverflow); - - // If class unloading is globally enabled, aggressive does unloading even with - // concurrent cycles. - if (ClassUnloading) { - SHENANDOAH_ERGO_OVERRIDE_DEFAULT(ShenandoahUnloadClassesFrequency, 1); - } -} - -void ShenandoahAggressiveOldHeuristics::choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset, - RegionData* data, size_t size, - size_t free) { - for (size_t idx = 0; idx < size; idx++) { - ShenandoahHeapRegion* r = data[idx]._region; - if (r->garbage() > 0) { - cset->add_region(r); - } - } -} - -bool ShenandoahAggressiveOldHeuristics::should_start_gc() { - log_info(gc)("Trigger: Start next cycle immediately"); - return true; -} - -bool ShenandoahAggressiveOldHeuristics::should_unload_classes() { - if (!can_unload_classes_normal()) return false; - if (has_metaspace_oom()) return true; - // Randomly unload classes with 50% chance. - return (os::random() & 1) == 1; -} diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveOldHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveOldHeuristics.hpp deleted file mode 100644 index 8f7b699ba9cfa..0000000000000 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveOldHeuristics.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHAGGRESSIVEOLDHEURISTICS_HPP -#define SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHAGGRESSIVEOLDHEURISTICS_HPP - -#include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp" - -class ShenandoahAggressiveOldHeuristics : public ShenandoahOldHeuristics { -public: - ShenandoahAggressiveOldHeuristics(ShenandoahGeneration* generation); - - virtual void choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset, - RegionData* data, size_t size, - size_t free); - - virtual bool should_start_gc(); - - virtual bool should_unload_classes(); - - virtual const char* name() { return "AggressiveOld"; } - virtual bool is_diagnostic() { return true; } - virtual bool is_experimental() { return false; } -}; - -#endif // SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHAGGRESSIVEOLDHEURISTICS_HPP diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactOldHeuristics.cpp deleted file mode 100644 index 3fb24256d1b29..0000000000000 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactOldHeuristics.cpp +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" - -#include "gc/shenandoah/shenandoahCollectionSet.hpp" -#include "gc/shenandoah/heuristics/shenandoahCompactOldHeuristics.hpp" -#include "gc/shenandoah/shenandoahFreeSet.hpp" -#include "gc/shenandoah/shenandoahGeneration.hpp" -#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" -#include "logging/log.hpp" -#include "logging/logTag.hpp" - -ShenandoahCompactOldHeuristics::ShenandoahCompactOldHeuristics(ShenandoahGeneration* generation) : - ShenandoahOldHeuristics(generation) { - SHENANDOAH_ERGO_ENABLE_FLAG(ExplicitGCInvokesConcurrent); - SHENANDOAH_ERGO_ENABLE_FLAG(ShenandoahImplicitGCInvokesConcurrent); - SHENANDOAH_ERGO_ENABLE_FLAG(ShenandoahUncommit); - SHENANDOAH_ERGO_ENABLE_FLAG(ShenandoahAlwaysClearSoftRefs); - SHENANDOAH_ERGO_OVERRIDE_DEFAULT(ShenandoahAllocationThreshold, 10); - SHENANDOAH_ERGO_OVERRIDE_DEFAULT(ShenandoahImmediateThreshold, 100); - SHENANDOAH_ERGO_OVERRIDE_DEFAULT(ShenandoahUncommitDelay, 1000); - SHENANDOAH_ERGO_OVERRIDE_DEFAULT(ShenandoahGuaranteedGCInterval, 30000); - SHENANDOAH_ERGO_OVERRIDE_DEFAULT(ShenandoahGarbageThreshold, 10); -} - -bool ShenandoahCompactOldHeuristics::should_start_gc() { - size_t max_capacity = _generation->max_capacity(); - size_t capacity = _generation->soft_max_capacity(); - size_t available = _generation->available(); - - // Make sure the code below treats available without the soft tail. - size_t soft_tail = max_capacity - capacity; - available = (available > soft_tail) ? (available - soft_tail) : 0; - - size_t threshold_bytes_allocated = capacity / 100 * ShenandoahAllocationThreshold; - size_t min_threshold = capacity / 100 * ShenandoahMinFreeThreshold; - - if (available < min_threshold) { - log_info(gc)("Trigger: Free (" SIZE_FORMAT "%s) is below minimum threshold (" SIZE_FORMAT "%s)", - byte_size_in_proper_unit(available), proper_unit_for_byte_size(available), - byte_size_in_proper_unit(min_threshold), proper_unit_for_byte_size(min_threshold)); - return true; - } - - size_t bytes_allocated = _generation->bytes_allocated_since_gc_start(); - if (bytes_allocated > threshold_bytes_allocated) { - log_info(gc)("Trigger: Allocated since last cycle (" SIZE_FORMAT "%s) is larger than allocation threshold (" SIZE_FORMAT "%s)", - byte_size_in_proper_unit(bytes_allocated), proper_unit_for_byte_size(bytes_allocated), - byte_size_in_proper_unit(threshold_bytes_allocated), proper_unit_for_byte_size(threshold_bytes_allocated)); - return true; - } - - return ShenandoahHeuristics::should_start_gc(); -} - -void ShenandoahCompactOldHeuristics::choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset, - RegionData* data, size_t size, - size_t actual_free) { - // Do not select too large CSet that would overflow the available free space - size_t max_cset = actual_free * 3 / 4; - - log_info(gc, ergo)("CSet Selection. Actual Free: " SIZE_FORMAT "%s, Max CSet: " SIZE_FORMAT "%s", - byte_size_in_proper_unit(actual_free), proper_unit_for_byte_size(actual_free), - byte_size_in_proper_unit(max_cset), proper_unit_for_byte_size(max_cset)); - - size_t threshold = ShenandoahHeapRegion::region_size_bytes() * ShenandoahGarbageThreshold / 100; - - size_t live_cset = 0; - for (size_t idx = 0; idx < size; idx++) { - ShenandoahHeapRegion* r = data[idx]._region; - size_t new_cset = live_cset + r->get_live_data_bytes(); - if (new_cset < max_cset && r->garbage() > threshold) { - live_cset = new_cset; - cset->add_region(r); - } - } -} diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactOldHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactOldHeuristics.hpp deleted file mode 100644 index 3297744ce8910..0000000000000 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactOldHeuristics.hpp +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHCOMPACTOLDHEURISTICS_HPP -#define SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHCOMPACTOLDHEURISTICS_HPP - -#include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp" - -class ShenandoahCompactOldHeuristics : public ShenandoahOldHeuristics { -public: - ShenandoahCompactOldHeuristics(ShenandoahGeneration* generation); - - virtual bool should_start_gc(); - - virtual void choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset, - RegionData* data, size_t size, - size_t actual_free); - - virtual const char* name() { return "CompactOld"; } - virtual bool is_diagnostic() { return false; } - virtual bool is_experimental() { return false; } -}; - -#endif // SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHCOMPACTOLDHEURISTICS_HPP diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp index 2ebf2ce912d97..623281e6e6be8 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp @@ -50,6 +50,7 @@ ShenandoahHeuristics::ShenandoahHeuristics(ShenandoahGeneration* generation) : _region_data(NULL), _degenerated_cycles_in_a_row(0), _successful_cycles_in_a_row(0), + _guaranteed_gc_interval(0), _cycle_start(os::elapsedTime()), _last_cycle_end(0), _gc_times_learned(0), @@ -221,11 +222,11 @@ bool ShenandoahHeuristics::should_start_gc() { return true; } - if (ShenandoahGuaranteedGCInterval > 0) { + if (_guaranteed_gc_interval > 0) { double last_time_ms = (os::elapsedTime() - _last_cycle_end) * 1000; - if (last_time_ms > ShenandoahGuaranteedGCInterval) { + if (last_time_ms > _guaranteed_gc_interval) { log_info(gc)("Trigger (%s): Time since last GC (%.0f ms) is larger than guaranteed interval (" UINTX_FORMAT " ms)", - _generation->name(), last_time_ms, ShenandoahGuaranteedGCInterval); + _generation->name(), last_time_ms, _guaranteed_gc_interval); return true; } } diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp index 65885b9a9779a..fe6f9da9b5df5 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp @@ -90,6 +90,8 @@ class ShenandoahHeuristics : public CHeapObj { uint _degenerated_cycles_in_a_row; uint _successful_cycles_in_a_row; + size_t _guaranteed_gc_interval; + double _cycle_start; double _last_cycle_end; @@ -122,6 +124,10 @@ class ShenandoahHeuristics : public CHeapObj { void clear_metaspace_oom() { _metaspace_oom.unset(); } bool has_metaspace_oom() const { return _metaspace_oom.is_set(); } + void set_guaranteed_gc_interval(size_t guaranteed_gc_interval) { + _guaranteed_gc_interval = guaranteed_gc_interval; + } + virtual void record_cycle_start(); virtual void record_cycle_end(); diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp index d4d4aac603bdf..41fd89ac8500c 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp @@ -27,18 +27,24 @@ #include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp" #include "utilities/quickSort.hpp" -ShenandoahOldHeuristics::ShenandoahOldHeuristics(ShenandoahGeneration* generation) : +ShenandoahOldHeuristics::ShenandoahOldHeuristics(ShenandoahGeneration* generation, ShenandoahHeuristics* trigger_heuristic) : ShenandoahHeuristics(generation), _old_collection_candidates(0), _next_old_collection_candidate(0), _hidden_old_collection_candidates(0), _hidden_next_old_collection_candidate(0), _old_coalesce_and_fill_candidates(0), - _first_coalesce_and_fill_candidate(0) + _first_coalesce_and_fill_candidate(0), + _trigger_heuristic(trigger_heuristic) { } bool ShenandoahOldHeuristics::prime_collection_set(ShenandoahCollectionSet* collection_set) { + if (unprocessed_old_collection_candidates() == 0) { + // no candidates for inclusion in collection set. + return false; + } + uint included_old_regions = 0; size_t evacuated_old_bytes = 0; @@ -46,12 +52,11 @@ bool ShenandoahOldHeuristics::prime_collection_set(ShenandoahCollectionSet* coll // The max_old_evacuation_bytes and promotion_budget_bytes constants represent a first // approximation to desired operating parameters. Eventually, these values should be determined // by heuristics and should adjust dynamically based on most current execution behavior. In the - // interrim, we may choose to offer command-line options to set the values of these configuration - // parameters. + // interim, we offer command-line options to set the values of these configuration parameters. // max_old_evacuation_bytes represents an "arbitrary" bound on how much evacuation effort is dedicated // to old-gen regions. - const size_t max_old_evacuation_bytes = (ShenandoahHeapRegion::region_size_bytes() * 8); + const size_t max_old_evacuation_bytes = (size_t) ((double)_generation->soft_max_capacity() / 100 * ShenandoahOldEvacReserve); // promotion_budget_bytes represents an "arbitrary" bound on how many bytes can be consumed by young-gen // objects promoted into old-gen memory. We need to avoid a scenario under which promotion of objects @@ -95,6 +100,10 @@ bool ShenandoahOldHeuristics::prime_collection_set(ShenandoahCollectionSet* coll } else excess_free_capacity = 0; + log_info(gc)("Choose old regions for mixed collection: excess capacity: " SIZE_FORMAT "%s, evacuation budget: " SIZE_FORMAT "%s", + byte_size_in_proper_unit((size_t) excess_free_capacity), proper_unit_for_byte_size(excess_free_capacity), + byte_size_in_proper_unit(old_evacuation_budget), proper_unit_for_byte_size(old_evacuation_budget)); + size_t remaining_old_evacuation_budget = old_evacuation_budget; // The number of old-gen regions that were selected as candidates for collection at the end of the most recent @@ -137,9 +146,8 @@ bool ShenandoahOldHeuristics::prime_collection_set(ShenandoahCollectionSet* coll } if (included_old_regions > 0) { - log_info(gc)("Old-gen piggyback evac (%llu regions, %llu bytes)", - (unsigned long long) included_old_regions, - (unsigned long long) evacuated_old_bytes); + log_info(gc)("Old-gen piggyback evac (" UINT32_FORMAT " regions, " SIZE_FORMAT " %s)", + included_old_regions, byte_size_in_proper_unit(evacuated_old_bytes), proper_unit_for_byte_size(evacuated_old_bytes)); } return (included_old_regions > 0); } @@ -155,10 +163,12 @@ bool ShenandoahOldHeuristics::choose_collection_set(ShenandoahCollectionSet* col void ShenandoahOldHeuristics::prepare_for_old_collections() { assert(_generation->generation_mode() == OLD, "This service only available for old-gc heuristics"); ShenandoahHeap* heap = ShenandoahHeap::heap(); - uint free_regions = 0; + size_t cand_idx = 0; size_t total_garbage = 0; size_t num_regions = heap->num_regions(); + size_t immediate_garbage = 0; + size_t immediate_regions = 0; RegionData* candidates = _region_data; for (size_t i = 0; i < num_regions; i++) { @@ -173,6 +183,8 @@ void ShenandoahOldHeuristics::prepare_for_old_collections() { if (region->is_regular()) { if (!region->has_live()) { region->make_trash_immediate(); + immediate_regions++; + immediate_garbage += garbage; } else { candidates[cand_idx]._region = region; candidates[cand_idx]._garbage = garbage; @@ -187,28 +199,31 @@ void ShenandoahOldHeuristics::prepare_for_old_collections() { size_t region_count = heap->trash_humongous_region_at(region); log_debug(gc)("Trashed " SIZE_FORMAT " regions for humongous object.", region_count); } + } else if (region->is_trash()) { + // Count humongous objects made into trash here. + immediate_regions++; + immediate_garbage += garbage; } } + // TODO: Consider not running mixed collects if we recovered some threshold percentage of memory from immediate garbage. + // This would be similar to young and global collections shortcutting evacuation, though we'd probably want a separate + // threshold for the old generation. + // Prioritize regions to select garbage-first regions QuickSort::sort(candidates, cand_idx, compare_by_garbage, false); - // Any old-gen region that contains (ShenandoahGarbageThreshold (default value 25))% garbage or more is to + // Any old-gen region that contains (ShenandoahOldGarbageThreshold (default value 25))% garbage or more is to // be evacuated. // - // TODO: it probably makes sense to define old-generation collection_threshold_garbage_percent differently - // than the young-gen ShenandoahGarbageThreshold. So a different command-line option might allow specification - // distinct values for each. Even better, allow collection_threshold_garbage_percent to be determined - // adaptively, by heuristics. - - const size_t collection_threshold_garbage_percent = ShenandoahGarbageThreshold; + // TODO: allow ShenandoahOldGarbageThreshold to be determined adaptively, by heuristics. - size_t region_size = ShenandoahHeapRegion::region_size_bytes(); + const size_t garbage_threshold = ShenandoahHeapRegion::region_size_bytes() * ShenandoahOldGarbageThreshold / 100; + size_t candidates_garbage = 0; for (size_t i = 0; i < cand_idx; i++) { - // Do approximate percent to avoid floating point math - size_t percent_garbage = candidates[i]._garbage * 100 / region_size; - - if (percent_garbage < collection_threshold_garbage_percent) { + candidates_garbage += candidates[i]._garbage; + if (candidates[i]._garbage < garbage_threshold) { + // Candidates are sorted in decreasing order of garbage, so no regions after this will be above the threshold _hidden_next_old_collection_candidate = 0; _hidden_old_collection_candidates = (uint)i; _first_coalesce_and_fill_candidate = (uint)i; @@ -216,9 +231,8 @@ void ShenandoahOldHeuristics::prepare_for_old_collections() { // Note that we do not coalesce and fill occupied humongous regions // HR: humongous regions, RR: regular regions, CF: coalesce and fill regions - log_info(gc)("Old-gen mark evac (%llu RR, %llu CF)", - (unsigned long long) (_hidden_old_collection_candidates), - (unsigned long long) _old_coalesce_and_fill_candidates); + log_info(gc)("Old-gen mark evac (" UINT32_FORMAT " RR, " UINT32_FORMAT " CF)", + _hidden_old_collection_candidates, _old_coalesce_and_fill_candidates); return; } } @@ -231,9 +245,13 @@ void ShenandoahOldHeuristics::prepare_for_old_collections() { // Note that we do not coalesce and fill occupied humongous regions // HR: humongous regions, RR: regular regions, CF: coalesce and fill regions - log_info(gc)("Old-gen mark evac (%llu RR, %llu CF)", - (unsigned long long) (_hidden_old_collection_candidates), - (unsigned long long) _old_coalesce_and_fill_candidates); + size_t collectable_garbage = immediate_garbage + candidates_garbage; + log_info(gc)("Old-gen mark evac (" UINT32_FORMAT " RR, " UINT32_FORMAT " CF), " + "Collectable Garbage: " SIZE_FORMAT "%s, " + "Immediate Garbage: " SIZE_FORMAT "%s", + _hidden_old_collection_candidates, _old_coalesce_and_fill_candidates, + byte_size_in_proper_unit(collectable_garbage), proper_unit_for_byte_size(collectable_garbage), + byte_size_in_proper_unit(immediate_garbage), proper_unit_for_byte_size(immediate_garbage)); } void ShenandoahOldHeuristics::start_old_evacuations() { @@ -242,7 +260,8 @@ void ShenandoahOldHeuristics::start_old_evacuations() { _old_collection_candidates = _hidden_old_collection_candidates; _next_old_collection_candidate = _hidden_next_old_collection_candidate; - _hidden_old_collection_candidates = 0;} + _hidden_old_collection_candidates = 0; +} uint ShenandoahOldHeuristics::unprocessed_old_collection_candidates() { @@ -280,7 +299,7 @@ bool ShenandoahOldHeuristics::should_defer_gc() { // Cannot start a new old-gen GC until previous one has finished. // // Future refinement: under certain circumstances, we might be more sophisticated about this choice. - // For example, we could choose to abandon the prevoius old collection before it has completed evacuations, + // For example, we could choose to abandon the previous old collection before it has completed evacuations, // but this would require that we coalesce and fill all garbage within unevacuated collection-set regions. return true; } @@ -296,3 +315,74 @@ void ShenandoahOldHeuristics::abandon_collection_candidates() { _first_coalesce_and_fill_candidate = 0; } +void ShenandoahOldHeuristics::record_cycle_start() { + _trigger_heuristic->record_cycle_start(); +} + +void ShenandoahOldHeuristics::record_cycle_end() { + _trigger_heuristic->record_cycle_end(); +} + +bool ShenandoahOldHeuristics::should_start_gc() { + if (should_defer_gc()) { + return false; + } + return _trigger_heuristic->should_start_gc(); +} + +bool ShenandoahOldHeuristics::should_degenerate_cycle() { + return _trigger_heuristic->should_degenerate_cycle(); +} + +void ShenandoahOldHeuristics::record_success_concurrent() { + _trigger_heuristic->record_success_concurrent(); +} + +void ShenandoahOldHeuristics::record_success_degenerated() { + _trigger_heuristic->record_success_degenerated(); +} + +void ShenandoahOldHeuristics::record_success_full() { + _trigger_heuristic->record_success_full(); +} + +void ShenandoahOldHeuristics::record_allocation_failure_gc() { + _trigger_heuristic->record_allocation_failure_gc(); +} + +void ShenandoahOldHeuristics::record_requested_gc() { + _trigger_heuristic->record_requested_gc(); +} + +bool ShenandoahOldHeuristics::can_unload_classes() { + return _trigger_heuristic->can_unload_classes(); +} + +bool ShenandoahOldHeuristics::can_unload_classes_normal() { + return _trigger_heuristic->can_unload_classes_normal(); +} + +bool ShenandoahOldHeuristics::should_unload_classes() { + return _trigger_heuristic->should_unload_classes(); +} + +const char* ShenandoahOldHeuristics::name() { + static char name[128]; + jio_snprintf(name, sizeof(name), "%s (OLD)", _trigger_heuristic->name()); + return name; +} + +bool ShenandoahOldHeuristics::is_diagnostic() { + return false; +} + +bool ShenandoahOldHeuristics::is_experimental() { + return true; +} + +void ShenandoahOldHeuristics::choose_collection_set_from_regiondata(ShenandoahCollectionSet* set, + ShenandoahHeuristics::RegionData* data, + size_t data_size, size_t free) { + ShouldNotReachHere(); +} + diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp index f55fe46b7fef2..33b6c8f68646f 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp @@ -33,7 +33,7 @@ class ShenandoahOldHeuristics : public ShenandoahHeuristics { -protected: +private: // if (_generation->generation_mode() == OLD) _old_collection_candidates // represent the number of regions selected for collection following the @@ -63,11 +63,19 @@ class ShenandoahOldHeuristics : public ShenandoahHeuristics { uint _old_coalesce_and_fill_candidates; uint _first_coalesce_and_fill_candidate; + // This can be the 'static' or 'adaptive' heuristic. + ShenandoahHeuristics* _trigger_heuristic; + // Prepare for evacuation of old-gen regions by capturing the mark results of a recently completed concurrent mark pass. void prepare_for_old_collections(); + + protected: + virtual void choose_collection_set_from_regiondata(ShenandoahCollectionSet* set, RegionData* data, size_t data_size, + size_t free) override; + public: - ShenandoahOldHeuristics(ShenandoahGeneration* generation); + ShenandoahOldHeuristics(ShenandoahGeneration* generation, ShenandoahHeuristics* trigger_heuristic); // Return true iff chosen collection set includes at least one old-gen HeapRegion. virtual bool choose_collection_set(ShenandoahCollectionSet* collection_set, ShenandoahOldHeuristics* old_heuristics); @@ -106,6 +114,36 @@ class ShenandoahOldHeuristics : public ShenandoahHeuristics { // held by this heuristic for supplying mixed collections. void abandon_collection_candidates(); + virtual void record_cycle_start() override; + + virtual void record_cycle_end() override; + + virtual bool should_start_gc() override; + + virtual bool should_degenerate_cycle() override; + + virtual void record_success_concurrent() override; + + virtual void record_success_degenerated() override; + + virtual void record_success_full() override; + + virtual void record_allocation_failure_gc() override; + + virtual void record_requested_gc() override; + + virtual bool can_unload_classes() override; + + virtual bool can_unload_classes_normal() override; + + virtual bool should_unload_classes() override; + + virtual const char* name() override; + + virtual bool is_diagnostic() override; + + virtual bool is_experimental() override; + }; #endif // SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHOLDHEURISTICS_HPP diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveOldHeuristics.cpp deleted file mode 100644 index eba854ae760d1..0000000000000 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveOldHeuristics.cpp +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" - -#include "gc/shenandoah/heuristics/shenandoahPassiveOldHeuristics.hpp" -#include "gc/shenandoah/shenandoahCollectionSet.hpp" -#include "gc/shenandoah/shenandoahHeap.inline.hpp" -#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" -#include "logging/log.hpp" -#include "logging/logTag.hpp" - -bool ShenandoahPassiveOldHeuristics::should_start_gc() { - // Never do concurrent GCs. - return false; -} - -bool ShenandoahPassiveOldHeuristics::should_unload_classes() { - // Always unload classes, if we can. - return can_unload_classes(); -} - -bool ShenandoahPassiveOldHeuristics::should_degenerate_cycle() { - // Always fail to Degenerated GC, if enabled - return ShenandoahDegeneratedGC; -} - -void ShenandoahPassiveOldHeuristics::choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset, - RegionData* data, size_t size, - size_t actual_free) { - assert(ShenandoahDegeneratedGC, "This path is only taken for Degenerated GC"); - - // Do not select too large CSet that would overflow the available free space. - // Take at least the entire evacuation reserve, and be free to overflow to free space. - size_t max_capacity = ShenandoahHeap::heap()->max_capacity(); - size_t available = MAX2(max_capacity / 100 * ShenandoahEvacReserve, actual_free); - size_t max_cset = (size_t)(available / ShenandoahEvacWaste); - - log_info(gc, ergo)("CSet Selection. Actual Free: " SIZE_FORMAT "%s, Max CSet: " SIZE_FORMAT "%s", - byte_size_in_proper_unit(actual_free), proper_unit_for_byte_size(actual_free), - byte_size_in_proper_unit(max_cset), proper_unit_for_byte_size(max_cset)); - - size_t threshold = ShenandoahHeapRegion::region_size_bytes() * ShenandoahGarbageThreshold / 100; - - size_t live_cset = 0; - for (size_t idx = 0; idx < size; idx++) { - ShenandoahHeapRegion* r = data[idx]._region; - size_t new_cset = live_cset + r->get_live_data_bytes(); - if (new_cset < max_cset && r->garbage() > threshold) { - live_cset = new_cset; - cset->add_region(r); - } - } -} diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveOldHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveOldHeuristics.hpp deleted file mode 100644 index 798a908fac313..0000000000000 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveOldHeuristics.hpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHPASSIVEOLDHEURISTICS_HPP -#define SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHPASSIVEOLDHEURISTICS_HPP - -#include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp" - -class ShenandoahPassiveOldHeuristics : public ShenandoahOldHeuristics { -public: - ShenandoahPassiveOldHeuristics(ShenandoahGeneration* generation) - : ShenandoahOldHeuristics(generation) {} - - virtual bool should_start_gc(); - - virtual bool should_unload_classes(); - - virtual bool should_degenerate_cycle(); - - virtual void choose_collection_set_from_regiondata(ShenandoahCollectionSet* set, - RegionData* data, size_t data_size, - size_t free); - - virtual const char* name() { return "PassiveOld"; } - virtual bool is_diagnostic() { return true; } - virtual bool is_experimental() { return false; } -}; - -#endif // SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHPASSIVEOLDHEURISTICS_HPP diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticOldHeuristics.cpp deleted file mode 100644 index 2f6b00a4c91b6..0000000000000 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticOldHeuristics.cpp +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" - -#include "gc/shenandoah/heuristics/shenandoahStaticOldHeuristics.hpp" -#include "gc/shenandoah/shenandoahCollectionSet.hpp" -#include "gc/shenandoah/shenandoahFreeSet.hpp" -#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" -#include "gc/shenandoah/shenandoahGeneration.hpp" -#include "logging/log.hpp" -#include "logging/logTag.hpp" - -ShenandoahStaticOldHeuristics::ShenandoahStaticOldHeuristics(ShenandoahGeneration* generation) : - ShenandoahOldHeuristics(generation) { - SHENANDOAH_ERGO_ENABLE_FLAG(ExplicitGCInvokesConcurrent); - SHENANDOAH_ERGO_ENABLE_FLAG(ShenandoahImplicitGCInvokesConcurrent); -} - -ShenandoahStaticOldHeuristics::~ShenandoahStaticOldHeuristics() {} - -bool ShenandoahStaticOldHeuristics::should_start_gc() { - size_t max_capacity = _generation->max_capacity(); - size_t capacity = _generation->soft_max_capacity(); - size_t available = _generation->available(); - - // Make sure the code below treats available without the soft tail. - size_t soft_tail = max_capacity - capacity; - available = (available > soft_tail) ? (available - soft_tail) : 0; - - size_t threshold_available = capacity / 100 * ShenandoahMinFreeThreshold; - - if (available < threshold_available) { - log_info(gc)("Trigger: Free (" SIZE_FORMAT "%s) is below minimum threshold (" SIZE_FORMAT "%s)", - byte_size_in_proper_unit(available), proper_unit_for_byte_size(available), - byte_size_in_proper_unit(threshold_available), proper_unit_for_byte_size(threshold_available)); - return true; - } - return ShenandoahHeuristics::should_start_gc(); -} - -void ShenandoahStaticOldHeuristics::choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset, - RegionData* data, size_t size, - size_t free) { - size_t threshold = ShenandoahHeapRegion::region_size_bytes() * ShenandoahGarbageThreshold / 100; - - for (size_t idx = 0; idx < size; idx++) { - ShenandoahHeapRegion* r = data[idx]._region; - if (r->garbage() > threshold) { - cset->add_region(r); - } - } -} diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticOldHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticOldHeuristics.hpp deleted file mode 100644 index b3aa3a6b90aed..0000000000000 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticOldHeuristics.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHSTATICOLDHEURISTICS_HPP -#define SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHSTATICOLDHEURISTICS_HPP - -#include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp" - -class ShenandoahStaticOldHeuristics : public ShenandoahOldHeuristics { -public: - ShenandoahStaticOldHeuristics(ShenandoahGeneration* generation); - - virtual ~ShenandoahStaticOldHeuristics(); - - virtual bool should_start_gc(); - - virtual void choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset, - RegionData* data, size_t size, - size_t free); - - virtual const char* name() { return "StaticOld"; } - virtual bool is_diagnostic() { return false; } - virtual bool is_experimental() { return false; } -}; - -#endif // SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHSTATICOLDHEURISTICS_HPP diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.cpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.cpp index 6011bca8622b4..0ab9296c8c69c 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.cpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.cpp @@ -24,13 +24,9 @@ #include "precompiled.hpp" #include "gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp" -#include "gc/shenandoah/heuristics/shenandoahAdaptiveOldHeuristics.hpp" #include "gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.hpp" -#include "gc/shenandoah/heuristics/shenandoahAggressiveOldHeuristics.hpp" #include "gc/shenandoah/heuristics/shenandoahCompactHeuristics.hpp" -#include "gc/shenandoah/heuristics/shenandoahCompactOldHeuristics.hpp" #include "gc/shenandoah/heuristics/shenandoahStaticHeuristics.hpp" -#include "gc/shenandoah/heuristics/shenandoahStaticOldHeuristics.hpp" #include "gc/shenandoah/shenandoahGeneration.hpp" #include "gc/shenandoah/mode/shenandoahMode.hpp" @@ -53,20 +49,3 @@ ShenandoahHeuristics* ShenandoahMode::initialize_heuristics(ShenandoahGeneration return NULL; } -ShenandoahOldHeuristics* ShenandoahMode::initialize_old_heuristics(ShenandoahGeneration* generation) const { - - assert(ShenandoahGCHeuristics != NULL, "ShenandoahGCHeuristics should not equal NULL"); - if (strcmp(ShenandoahGCHeuristics, "aggressive") == 0) { - return new ShenandoahAggressiveOldHeuristics(generation); - } else if (strcmp(ShenandoahGCHeuristics, "static") == 0) { - return new ShenandoahStaticOldHeuristics(generation); - } else if (strcmp(ShenandoahGCHeuristics, "adaptive") == 0) { - return new ShenandoahAdaptiveOldHeuristics(generation); - } else if (strcmp(ShenandoahGCHeuristics, "compact") == 0) { - return new ShenandoahCompactOldHeuristics(generation); - } else { - vm_exit_during_initialization("Unknown -XX:ShenandoahGCHeuristics option"); - ShouldNotReachHere(); - return NULL; - } -} diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.hpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.hpp index 5c74eb74d71df..46e2079d593ea 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.hpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.hpp @@ -53,7 +53,6 @@ class ShenandoahMode : public CHeapObj { public: virtual void initialize_flags() const = 0; virtual ShenandoahHeuristics* initialize_heuristics(ShenandoahGeneration* generation) const; - virtual ShenandoahOldHeuristics* initialize_old_heuristics(ShenandoahGeneration* generation) const; virtual const char* name() = 0; virtual bool is_diagnostic() = 0; virtual bool is_experimental() = 0; diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.cpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.cpp index c7ec1c01bd5d1..2e89157047319 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.cpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.cpp @@ -25,7 +25,6 @@ #include "precompiled.hpp" #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" #include "gc/shenandoah/heuristics/shenandoahPassiveHeuristics.hpp" -#include "gc/shenandoah/heuristics/shenandoahPassiveOldHeuristics.hpp" #include "gc/shenandoah/mode/shenandoahPassiveMode.hpp" #include "logging/log.hpp" #include "logging/logTag.hpp" @@ -62,8 +61,3 @@ ShenandoahHeuristics* ShenandoahPassiveMode::initialize_heuristics(ShenandoahGen ShouldNotReachHere(); return NULL; } - -ShenandoahOldHeuristics* ShenandoahPassiveMode::initialize_old_heuristics(ShenandoahGeneration* generation) const { - assert(ShenandoahGCHeuristics != NULL, "ShenandoahGCHeuristics should not equal NULL"); - return new ShenandoahPassiveOldHeuristics(generation); -} diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.hpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.hpp index ec5e5f85f817a..7b3f6da92bcb2 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.hpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.hpp @@ -31,7 +31,6 @@ class ShenandoahPassiveMode : public ShenandoahMode { public: virtual void initialize_flags() const; virtual ShenandoahHeuristics* initialize_heuristics(ShenandoahGeneration* generation) const; - virtual ShenandoahOldHeuristics* initialize_old_heuristics(ShenandoahGeneration* generation) const; virtual const char* name() { return "Passive"; } virtual bool is_diagnostic() { return true; } virtual bool is_experimental() { return false; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index d690b0d4ba0d1..f4c09357b91b1 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -103,15 +103,9 @@ void ShenandoahGeneration::confirm_heuristics_mode() { } } -ShenandoahOldHeuristics* ShenandoahGeneration::initialize_old_heuristics(ShenandoahMode* gc_mode) { - ShenandoahOldHeuristics* old_heuristics = gc_mode->initialize_old_heuristics(this); - _heuristics = old_heuristics; - confirm_heuristics_mode(); - return old_heuristics; -} - ShenandoahHeuristics* ShenandoahGeneration::initialize_heuristics(ShenandoahMode* gc_mode) { _heuristics = gc_mode->initialize_heuristics(this); + _heuristics->set_guaranteed_gc_interval(ShenandoahGuaranteedGCInterval); confirm_heuristics_mode(); return _heuristics; } @@ -282,7 +276,8 @@ ShenandoahGeneration::ShenandoahGeneration(GenerationMode generation_mode, _task_queues(new ShenandoahObjToScanQueueSet(max_workers)), _ref_processor(new ShenandoahReferenceProcessor(MAX2(max_workers, 1U))), _affiliated_region_count(0), _used(0), _bytes_allocated_since_gc_start(0), - _max_capacity(max_capacity), _soft_max_capacity(soft_max_capacity) { + _max_capacity(max_capacity), _soft_max_capacity(soft_max_capacity), + _heuristics(nullptr) { _is_marking_complete.set(); assert(max_workers > 0, "At least one queue"); for (uint i = 0; i < max_workers; ++i) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index 32bb4e0bc6926..902273bbf4d5b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -38,7 +38,6 @@ class ShenandoahReferenceProcessor; class ShenandoahGeneration : public CHeapObj { private: GenerationMode const _generation_mode; - ShenandoahHeuristics* _heuristics; // Marking task queues and completeness ShenandoahObjToScanQueueSet* _task_queues; @@ -54,7 +53,8 @@ class ShenandoahGeneration : public CHeapObj { size_t _max_capacity; size_t _soft_max_capacity; -public: + ShenandoahHeuristics* _heuristics; + public: ShenandoahGeneration(GenerationMode generation_mode, uint max_workers, size_t max_capacity, size_t soft_max_capacity); ~ShenandoahGeneration(); @@ -66,9 +66,7 @@ class ShenandoahGeneration : public CHeapObj { virtual const char* name() const = 0; - ShenandoahHeuristics* initialize_heuristics(ShenandoahMode* gc_mode); - - ShenandoahOldHeuristics* initialize_old_heuristics(ShenandoahMode* gc_mode); + virtual ShenandoahHeuristics* initialize_heuristics(ShenandoahMode* gc_mode); virtual size_t soft_max_capacity() const { return _soft_max_capacity; } virtual size_t max_capacity() const { return _max_capacity; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 7e909e2499c59..74c3b52d9b841 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -485,9 +485,11 @@ void ShenandoahHeap::initialize_heuristics() { _gc_mode->name())); } - _old_heuristics = _old_generation->initialize_old_heuristics(_gc_mode); _global_generation->initialize_heuristics(_gc_mode); - _young_generation->initialize_heuristics(_gc_mode); + if (mode()->is_generational()) { + _young_generation->initialize_heuristics(_gc_mode); + _old_generation->initialize_heuristics(_gc_mode); + } } #ifdef _MSC_VER @@ -498,7 +500,6 @@ void ShenandoahHeap::initialize_heuristics() { ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) : CollectedHeap(), _gc_generation(NULL), - _old_heuristics(nullptr), _mixed_evac(false), _initial_size(0), _used(0), @@ -625,8 +626,13 @@ void ShenandoahHeap::post_initialize() { JFR_ONLY(ShenandoahJFRSupport::register_jfr_type_serializers()); } + +ShenandoahOldHeuristics* ShenandoahHeap::old_heuristics() { + return (ShenandoahOldHeuristics*) _old_generation->heuristics(); +} + bool ShenandoahHeap::doing_mixed_evacuations() { - return (_old_heuristics->unprocessed_old_collection_candidates() > 0); + return old_heuristics()->unprocessed_old_collection_candidates() > 0; } bool ShenandoahHeap::is_gc_generation_young() const { @@ -940,7 +946,7 @@ void ShenandoahHeap::retire_plab(PLAB* plab) { void ShenandoahHeap::cancel_mixed_collections() { assert(_old_generation != NULL, "Should only have mixed collections in generation mode."); - _old_heuristics->abandon_collection_candidates(); + old_heuristics()->abandon_collection_candidates(); } void ShenandoahHeap::coalesce_and_fill_old_regions() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index 2a2506cf28166..6c2494b6b8513 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -150,7 +150,6 @@ class ShenandoahHeap : public CollectedHeap { private: ShenandoahHeapLock _lock; ShenandoahGeneration* _gc_generation; - ShenandoahOldHeuristics* _old_heuristics; bool _mixed_evac; // true iff most recent evac included at least one old-gen HeapRegion public: @@ -171,9 +170,7 @@ class ShenandoahHeap : public CollectedHeap { _mixed_evac = mixed_evac; } - ShenandoahOldHeuristics* old_heuristics() { - return _old_heuristics; - } + ShenandoahOldHeuristics* old_heuristics(); bool doing_mixed_evacuations(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp index 891b42b2693ef..f1e3a68c1c573 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp @@ -22,9 +22,14 @@ * */ + #include "precompiled.hpp" #include "gc/shared/strongRootsScope.hpp" +#include "gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp" +#include "gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.hpp" +#include "gc/shenandoah/heuristics/shenandoahCompactHeuristics.hpp" +#include "gc/shenandoah/heuristics/shenandoahStaticHeuristics.hpp" #include "gc/shenandoah/shenandoahAsserts.hpp" #include "gc/shenandoah/shenandoahFreeSet.hpp" #include "gc/shenandoah/shenandoahHeap.hpp" @@ -198,3 +203,22 @@ bool ShenandoahOldGeneration::prepare_regions_and_collection_set(bool concurrent } return false; } + +ShenandoahHeuristics* ShenandoahOldGeneration::initialize_heuristics(ShenandoahMode* gc_mode) { + assert(ShenandoahOldGCHeuristics != NULL, "ShenandoahOldGCHeuristics should not equal NULL"); + ShenandoahHeuristics* trigger; + if (strcmp(ShenandoahOldGCHeuristics, "static") == 0) { + trigger = new ShenandoahStaticHeuristics(this); + } else if (strcmp(ShenandoahOldGCHeuristics, "adaptive") == 0) { + trigger = new ShenandoahAdaptiveHeuristics(this); + } else if (strcmp(ShenandoahOldGCHeuristics, "compact") == 0) { + trigger = new ShenandoahCompactHeuristics(this); + } else { + vm_exit_during_initialization("Unknown -XX:ShenandoahOldGCHeuristics option (must be one of: static, adaptive, compact)"); + ShouldNotReachHere(); + return NULL; + } + trigger->set_guaranteed_gc_interval(ShenandoahGuaranteedOldGCInterval); + _heuristics = new ShenandoahOldHeuristics(this, trigger); + return _heuristics; +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp index 94151d5c374da..9b146051f895e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp @@ -48,6 +48,8 @@ class ShenandoahOldGeneration : public ShenandoahGeneration { bool prepare_regions_and_collection_set(bool concurrent) override; + virtual ShenandoahHeuristics* initialize_heuristics(ShenandoahMode* gc_mode) override; + // We leave the SATB barrier on for the entirety of the old generation // marking phase. In some cases, this can cause a write to a perfectly // reachable oop to enqueue a pointer that later becomes garbage (because diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.cpp index cac4a3384d2fb..a80a890e4f1e1 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.cpp @@ -43,7 +43,7 @@ ShenandoahRegulatorThread::ShenandoahRegulatorThread(ShenandoahControlThread* co _last_sleep_adjust_time(os::elapsedTime()) { ShenandoahHeap* heap = ShenandoahHeap::heap(); - _old_heuristics = heap->old_heuristics(); + _old_heuristics = get_heuristics(heap->old_generation()); _young_heuristics = get_heuristics(heap->young_generation()); _global_heuristics = get_heuristics(heap->global_generation()); @@ -134,7 +134,7 @@ void ShenandoahRegulatorThread::regulator_sleep() { } bool ShenandoahRegulatorThread::start_old_cycle() { - return !_old_heuristics->should_defer_gc() && _old_heuristics->should_start_gc() && _control_thread->request_concurrent_gc(OLD); + return _old_heuristics->should_start_gc() && _control_thread->request_concurrent_gc(OLD); } bool ShenandoahRegulatorThread::start_young_cycle() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.hpp b/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.hpp index 664b24f8a0d93..b29ba8f0a06a3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.hpp @@ -26,11 +26,12 @@ #include "gc/shared/concurrentGCThread.hpp" #include "gc/shared/gcCause.hpp" -#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" -#include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp" -#include "gc/shenandoah/shenandoahHeap.hpp" +#include "gc/shenandoah/shenandoahSharedVariables.hpp" #include "runtime/mutex.hpp" +class ShenandoahHeuristics; +class ShenandoahControlThread; + /* * The purpose of this class (and thread) is to allow us to continue * to evaluate heuristics during a garbage collection. This is necessary @@ -75,7 +76,7 @@ class ShenandoahRegulatorThread: public ConcurrentGCThread { ShenandoahSharedFlag _heap_changed; ShenandoahControlThread* _control_thread; ShenandoahHeuristics* _young_heuristics; - ShenandoahOldHeuristics* _old_heuristics; + ShenandoahHeuristics* _old_heuristics; ShenandoahHeuristics* _global_heuristics; int _sleep; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp index 17e818f989392..d350d6ae05ddd 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp @@ -610,7 +610,6 @@ ShenandoahScanRemembered::process_clusters(size_t first_cluster, // collected and their memory repurposed, and because zombie objects might refer to objects that are themselves dead. ShenandoahHeap* heap = ShenandoahHeap::heap(); - ShenandoahOldHeuristics* old_heuristics = heap->old_heuristics(); ShenandoahMarkingContext* ctx; if (heap->doing_mixed_evacuations()) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp index 80b2851979906..52dc5d391d490 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp @@ -77,6 +77,12 @@ " compact - run GC more frequently and with deeper targets to " \ "free up more memory.") \ \ + product(ccstr, ShenandoahOldGCHeuristics, "adaptive", \ + "Similar to ShenandoahGCHeuristics, but applied to the old " \ + "generation. This configuration is only used to trigger old " \ + "collections and does not change how regions are selected " \ + "for collection.") \ + \ product(uintx, ShenandoahUnloadClassesFrequency, 1, EXPERIMENTAL, \ "Unload the classes every Nth cycle. Normally affects concurrent "\ "GC cycles, as degenerated and full GCs would try to unload " \ @@ -90,6 +96,11 @@ "collector accepts. In percents of heap region size.") \ range(0,100) \ \ + product(uintx, ShenandoahOldGarbageThreshold, 25, EXPERIMENTAL, \ + "How much garbage an old region has to contain before it would " \ + "be taken for collection.") \ + range(0,100) \ + \ product(uintx, ShenandoahInitFreeThreshold, 70, EXPERIMENTAL, \ "How much heap should be free before some heuristics trigger the "\ "initial (learning) cycles. Affects cycle frequency on startup " \ @@ -164,6 +175,11 @@ "time from active application. Time is in milliseconds. " \ "Setting this to 0 disables the feature.") \ \ + product(uintx, ShenandoahGuaranteedOldGCInterval, 10*60*1000, EXPERIMENTAL, \ + "Run a collection of the old generation at least this often. " \ + "Heuristics may trigger collections more frequently. Time is in " \ + "milliseconds. Setting this to 0 disables the feature.") \ + \ product(bool, ShenandoahAlwaysClearSoftRefs, false, EXPERIMENTAL, \ "Unconditionally clear soft references, instead of using any " \ "other cleanup policy. This minimizes footprint at expense of" \ @@ -249,6 +265,12 @@ "reserve/waste is incorrect, at the risk that application " \ "runs out of memory too early.") \ \ + product(double, ShenandoahOldEvacReserve, 5.0, EXPERIMENTAL, \ + "How much of old generation to withhold from evacuations. " \ + "Larger values will result in fewer live objects being " \ + "evacuated in the old generation.") \ + range(1,100) \ + \ product(bool, ShenandoahPacing, true, EXPERIMENTAL, \ "Pace application allocations to give GC chance to start " \ "and complete before allocation failure is reached.") \ From 6a1eebb97378db11fc269c4ffce550bea2567777 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Wed, 6 Oct 2021 18:22:47 +0000 Subject: [PATCH 075/254] Improve handling of promotion and old generation evacuation failures Reviewed-by: kdnilsen --- .../heuristics/shenandoahOldHeuristics.cpp | 33 +++++++++++-------- .../heuristics/shenandoahOldHeuristics.hpp | 11 +++++-- .../gc/shenandoah/shenandoahControlThread.cpp | 7 ++-- .../share/gc/shenandoah/shenandoahHeap.cpp | 10 ++++++ .../share/gc/shenandoah/shenandoahHeap.hpp | 5 +++ .../gc/shenandoah/shenandoahHeap.inline.hpp | 21 +++++++++--- 6 files changed, 65 insertions(+), 22 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp index 41fd89ac8500c..ba566a76f1988 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp @@ -294,18 +294,6 @@ void ShenandoahOldHeuristics::get_coalesce_and_fill_candidates(ShenandoahHeapReg } } -bool ShenandoahOldHeuristics::should_defer_gc() { - if (unprocessed_old_collection_candidates() > 0) { - // Cannot start a new old-gen GC until previous one has finished. - // - // Future refinement: under certain circumstances, we might be more sophisticated about this choice. - // For example, we could choose to abandon the previous old collection before it has completed evacuations, - // but this would require that we coalesce and fill all garbage within unevacuated collection-set regions. - return true; - } - return false; -} - void ShenandoahOldHeuristics::abandon_collection_candidates() { _old_collection_candidates = 0; _next_old_collection_candidate = 0; @@ -315,7 +303,12 @@ void ShenandoahOldHeuristics::abandon_collection_candidates() { _first_coalesce_and_fill_candidate = 0; } +void ShenandoahOldHeuristics::handle_promotion_failure() { + _promotion_failed = true; +} + void ShenandoahOldHeuristics::record_cycle_start() { + _promotion_failed = false; _trigger_heuristic->record_cycle_start(); } @@ -324,9 +317,23 @@ void ShenandoahOldHeuristics::record_cycle_end() { } bool ShenandoahOldHeuristics::should_start_gc() { - if (should_defer_gc()) { + // Cannot start a new old-gen GC until previous one has finished. + // + // Future refinement: under certain circumstances, we might be more sophisticated about this choice. + // For example, we could choose to abandon the previous old collection before it has completed evacuations, + // but this would require that we coalesce and fill all garbage within unevacuated collection-set regions. + if (unprocessed_old_collection_candidates() > 0) { return false; } + + // If there's been a promotion failure (and we don't have regions already scheduled for evacuation), + // start a new old generation collection. + if (_promotion_failed) { + log_info(gc)("Trigger: Promotion Failure"); + return true; + } + + // Otherwise, defer to configured heuristic for gc trigger. return _trigger_heuristic->should_start_gc(); } diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp index 33b6c8f68646f..a20f6df0790ca 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp @@ -66,10 +66,12 @@ class ShenandoahOldHeuristics : public ShenandoahHeuristics { // This can be the 'static' or 'adaptive' heuristic. ShenandoahHeuristics* _trigger_heuristic; + // Flag is set when promotion failure is detected (by gc thread), cleared when old generation collection begins (by control thread) + volatile bool _promotion_failed; + // Prepare for evacuation of old-gen regions by capturing the mark results of a recently completed concurrent mark pass. void prepare_for_old_collections(); - protected: virtual void choose_collection_set_from_regiondata(ShenandoahCollectionSet* set, RegionData* data, size_t data_size, size_t free) override; @@ -108,12 +110,15 @@ class ShenandoahOldHeuristics : public ShenandoahHeuristics { // end of the array. void get_coalesce_and_fill_candidates(ShenandoahHeapRegion** buffer); - bool should_defer_gc(); - // If a GLOBAL gc occurs, it will collect the entire heap which invalidates any collection candidates being // held by this heuristic for supplying mixed collections. void abandon_collection_candidates(); + // Notify the heuristic of promotion failures. The promotion attempt will be skipped and the object will + // be evacuated into the young generation. The collection should complete normally, but we want to schedule + // an old collection as soon as possible. + void handle_promotion_failure(); + virtual void record_cycle_start() override; virtual void record_cycle_end() override; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index bd0583bcc9b63..1c578793269af 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -142,8 +142,10 @@ void ShenandoahControlThread::run_service() { ShenandoahHeuristics* heuristics = _degen_generation->heuristics(); generation = _degen_generation->generation_mode(); + bool old_gen_evacuation_failed = heap->clear_old_evacuation_failure(); - if (ShenandoahDegeneratedGC && heuristics->should_degenerate_cycle()) { + // Do not bother with degenerated cycle if old generation evacuation failed. + if (ShenandoahDegeneratedGC && heuristics->should_degenerate_cycle() && !old_gen_evacuation_failed) { heuristics->record_allocation_failure_gc(); policy->record_alloc_failure_to_degenerated(degen_point); set_gc_mode(stw_degenerated); @@ -370,7 +372,8 @@ void ShenandoahControlThread::run_service() { last_shrink_time = current; } - { + // Don't wait around if there was an allocation failure - start the next cycle immediately. + if (!is_alloc_failure_gc()) { // The timed wait is necessary because this thread has a responsibility to send // 'alloc_words' to the pacer when it does not perform a GC. MonitorLocker lock(&_control_lock, Mutex::_no_safepoint_check_flag); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 74c3b52d9b841..5af27d6d6bbdd 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -824,6 +824,16 @@ void ShenandoahHeap::handle_old_evacuation(HeapWord* obj, size_t words, bool pro } } +void ShenandoahHeap::handle_old_evacuation_failure() { + if (_old_gen_oom_evac.try_set()) { + log_info(gc)("Old gen evac failure."); + } +} + +void ShenandoahHeap::handle_promotion_failure() { + old_heuristics()->handle_promotion_failure(); +} + HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) { // New object should fit the GCLAB size size_t min_size = MAX2(size, PLAB::min_size()); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index 6c2494b6b8513..907902112cdb1 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -668,9 +668,12 @@ class ShenandoahHeap : public CollectedHeap { private: ShenandoahCollectionSet* _collection_set; ShenandoahEvacOOMHandler _oom_evac_handler; + ShenandoahSharedFlag _old_gen_oom_evac; inline oop try_evacuate_object(oop src, Thread* thread, ShenandoahHeapRegion* from_region, ShenandoahRegionAffiliation target_gen); void handle_old_evacuation(HeapWord* obj, size_t words, bool promotion); + void handle_old_evacuation_failure(); + void handle_promotion_failure(); public: static address in_cset_fast_test_addr(); @@ -691,6 +694,8 @@ class ShenandoahHeap : public CollectedHeap { inline void enter_evacuation(Thread* t); inline void leave_evacuation(Thread* t); + inline bool clear_old_evacuation_failure(); + // ---------- Generational support // private: diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp index 3b39da5c47389..f821df1aefb91 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp @@ -315,7 +315,8 @@ inline HeapWord* ShenandoahHeap::allocate_from_plab(Thread* thread, size_t size) } inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) { - if (ShenandoahThreadLocalData::is_oom_during_evac(Thread::current())) { + assert(thread == Thread::current(), "Expected thread parameter to be current thread."); + if (ShenandoahThreadLocalData::is_oom_during_evac(thread)) { // This thread went through the OOM during evac protocol and it is safe to return // the forward pointer. It must not attempt to evacuate any more. return ShenandoahBarrierSet::resolve_forwarded(p); @@ -388,9 +389,17 @@ inline oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, Shenandoah #endif if (copy == NULL) { - if (target_gen == OLD_GENERATION && from_region->affiliation() == YOUNG_GENERATION) { - // TODO: Inform old generation heuristic of promotion failure - return NULL; + if (target_gen == OLD_GENERATION) { + assert(mode()->is_generational(), "Should only be here in generational mode."); + if (from_region->is_young()) { + // Signal that promotion failed. Will evacuate this old object somewhere in young gen. + handle_promotion_failure(); + return NULL; + } else { + // Remember that evacuation to old gen failed. We'll want to trigger a full gc to recover from this + // after the evacuation threads have finished. + handle_old_evacuation_failure(); + } } control_thread()->handle_alloc_failure_evac(size); @@ -466,6 +475,10 @@ void ShenandoahHeap::increase_object_age(oop obj, uint additional_age) { } } +inline bool ShenandoahHeap::clear_old_evacuation_failure() { + return _old_gen_oom_evac.try_unset(); +} + inline bool ShenandoahHeap::is_old(oop obj) const { return is_gc_generation_young() && is_in_old(obj); } From a910ee86ca788a502d02a87a8723d818fe0d93c3 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Wed, 6 Oct 2021 19:02:30 +0000 Subject: [PATCH 076/254] Age cycle periodically Reviewed-by: wkemper --- .../share/gc/shenandoah/shenandoahConcurrentGC.cpp | 7 ++++++- .../share/gc/shenandoah/shenandoahControlThread.cpp | 6 ++++++ .../share/gc/shenandoah/shenandoahDegeneratedGC.cpp | 4 ++++ src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp | 4 ++++ src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp | 3 +++ src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp | 8 +++++++- .../share/gc/shenandoah/shenandoahMarkClosures.cpp | 2 +- src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp | 7 ++++++- 8 files changed, 37 insertions(+), 4 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index c403fe277eb11..b894daf38d02a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -134,7 +134,8 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) { } // Final mark might have reclaimed some immediate garbage, kick cleanup to reclaim - // the space. This would be the last action if there is nothing to evacuate. + // the space. This would be the last action if there is nothing to evacuate. Note that + // we will not age young-gen objects in the case that we skip evacuation. entry_cleanup_early(); { @@ -1034,6 +1035,10 @@ void ShenandoahConcurrentGC::op_final_updaterefs() { heap->set_update_refs_in_progress(false); heap->set_has_forwarded_objects(false); + // Aging_cycle is only relevant during evacuation cycle for individual objects and during final mark for + // entire regions. Both of these relevant operations occur before final update refs. + heap->set_aging_cycle(false); + if (ShenandoahVerify) { heap->verifier()->verify_after_updaterefs(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index 1c578793269af..2e88f0fef789f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -92,6 +92,7 @@ void ShenandoahControlThread::run_service() { GCCause::Cause default_cause = GCCause::_shenandoah_concurrent_gc; double last_shrink_time = os::elapsedTime(); + uint age_period = 0; // Shrink period avoids constantly polling regions for shrinking. // Having a period 10x lower than the delay would mean we hit the @@ -249,9 +250,14 @@ void ShenandoahControlThread::run_service() { heap->free_set()->log_status(); } + heap->set_aging_cycle(false); { switch (_mode) { case concurrent_normal: { + if ((generation == YOUNG) && (age_period-- == 0)) { + heap->set_aging_cycle(true); + age_period = ShenandoahAgingCyclePeriod - 1; + } service_concurrent_normal_cycle(heap, generation, cause); break; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp index b406969da14d4..4d98d856e6d6d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp @@ -68,6 +68,10 @@ void ShenandoahDegenGC::entry_degenerated() { EventMark em("%s", msg); ShenandoahHeap* const heap = ShenandoahHeap::heap(); + // In case degenerated GC preempted evacuation or update-refs, clear the aging cycle now. No harm in clearing it + // redundantly if it is already clear. We don't age during degenerated cycles. + heap->set_aging_cycle(false); + ShenandoahWorkerScope scope(heap->workers(), ShenandoahWorkerPolicy::calc_workers_for_stw_degenerated(), "stw degenerated gc"); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 5af27d6d6bbdd..282a689a80017 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -1921,6 +1921,10 @@ void ShenandoahHeap::set_concurrent_old_mark_in_progress(bool in_progress) { manage_satb_barrier(in_progress); } +void ShenandoahHeap::set_aging_cycle(bool in_progress) { + _is_aging_cycle.set_cond(in_progress); +} + void ShenandoahHeap::manage_satb_barrier(bool active) { if (is_concurrent_mark_in_progress()) { // Ignore request to deactivate barrier while concurrent mark is in progress. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index 907902112cdb1..a23450c94bc07 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -351,6 +351,7 @@ class ShenandoahHeap : public CollectedHeap { void set_has_forwarded_objects(bool cond); void set_concurrent_strong_root_in_progress(bool cond); void set_concurrent_weak_root_in_progress(bool cond); + void set_aging_cycle(bool cond); inline bool is_stable() const; inline bool is_idle() const; @@ -367,6 +368,7 @@ class ShenandoahHeap : public CollectedHeap { inline bool is_stw_gc_in_progress() const; inline bool is_concurrent_strong_root_in_progress() const; inline bool is_concurrent_weak_root_in_progress() const; + inline bool is_aging_cycle() const; private: void manage_satb_barrier(bool active); @@ -493,6 +495,7 @@ class ShenandoahHeap : public CollectedHeap { // ---------- Class Unloading // private: + ShenandoahSharedFlag _is_aging_cycle; ShenandoahSharedFlag _unload_classes; ShenandoahUnload _unloader; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp index f821df1aefb91..6fab11347f01a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp @@ -421,7 +421,9 @@ inline oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, Shenandoah if (target_gen == OLD_GENERATION) { handle_old_evacuation(copy, size, from_region->is_young()); } else if (target_gen == YOUNG_GENERATION) { - ShenandoahHeap::increase_object_age(copy_val, from_region->age() + 1); + if (is_aging_cycle()) { + ShenandoahHeap::increase_object_age(copy_val, from_region->age() + 1); + } } else { ShouldNotReachHere(); } @@ -554,6 +556,10 @@ inline bool ShenandoahHeap::is_concurrent_weak_root_in_progress() const { return _gc_state.is_set(WEAK_ROOTS); } +inline bool ShenandoahHeap::is_aging_cycle() const { + return _is_aging_cycle.is_set(); +} + template inline void ShenandoahHeap::marked_object_iterate(ShenandoahHeapRegion* region, T* cl) { marked_object_iterate(region, cl, region->top()); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.cpp index 603ad9e819f5a..8d2c5492f0c59 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.cpp @@ -71,7 +71,7 @@ void ShenandoahFinalMarkUpdateRegionStateClosure::heap_region_do(ShenandoahHeapR // There have been allocations in this region since the start of the cycle. // Any objects new to this region must not assimilate elevated age. r->reset_age(); - } else { + } else if (ShenandoahHeap::heap()->is_aging_cycle()) { r->increment_age(); } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp index 52dc5d391d490..3ff9e6278550f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp @@ -427,7 +427,12 @@ \ product(bool, ShenandoahAllowOldMarkingPreemption, true, DIAGNOSTIC, \ "Allow young generation collections to suspend concurrent" \ - " marking in the old generation.") + " marking in the old generation.") \ + \ + product(uintx, ShenandoahAgingCyclePeriod, 1, EXPERIMENTAL, \ + "With generational mode, increment the age of objects and" \ + "regions each time this many young-gen GC cycles are completed.") + // end of GC_SHENANDOAH_FLAGS // 2^ShenandoahTenuredRegionUsageBiasLogBase2 is 128 From d59d18efd0f5ecb2790c59793afb878383ca61e2 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Thu, 7 Oct 2021 21:15:38 +0000 Subject: [PATCH 077/254] Add an option to stream region sampling data to a file Co-authored-by: Christian Young Reviewed-by: rkennke --- .../shenandoahHeapRegionCounters.cpp | 28 ++- .../shenandoahHeapRegionCounters.hpp | 5 + .../gc/shenandoah/shenandoahLogFileOutput.cpp | 216 ++++++++++++++++++ .../gc/shenandoah/shenandoahLogFileOutput.hpp | 78 +++++++ .../gc/shenandoah/shenandoah_globals.hpp | 12 +- .../share/logging/logFileStreamOutput.cpp | 14 -- .../share/logging/logFileStreamOutput.hpp | 14 ++ .../shenandoah/TestRegionSamplingLogging.java | 58 +++++ 8 files changed, 400 insertions(+), 25 deletions(-) create mode 100644 src/hotspot/share/gc/shenandoah/shenandoahLogFileOutput.cpp create mode 100644 src/hotspot/share/gc/shenandoah/shenandoahLogFileOutput.hpp create mode 100644 test/hotspot/jtreg/gc/shenandoah/TestRegionSamplingLogging.java diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp index 268516d9633e9..f31e5495a1efa 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp @@ -29,12 +29,14 @@ #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" #include "gc/shenandoah/shenandoahHeapRegionSet.hpp" #include "gc/shenandoah/shenandoahHeapRegionCounters.hpp" +#include "gc/shenandoah/shenandoahLogFileOutput.hpp" #include "memory/resourceArea.hpp" #include "runtime/atomic.hpp" #include "runtime/perfData.inline.hpp" +#include "utilities/defaultStream.hpp" ShenandoahHeapRegionCounters::ShenandoahHeapRegionCounters() : - _last_sample_millis(0) + _last_sample_millis(0), _log_file(nullptr) { if (UsePerfData && ShenandoahRegionSampling) { EXCEPTION_MARK; @@ -43,7 +45,7 @@ ShenandoahHeapRegionCounters::ShenandoahHeapRegionCounters() : size_t num_regions = heap->num_regions(); const char* cns = PerfDataManager::name_space("shenandoah", "regions"); _name_space = NEW_C_HEAP_ARRAY(char, strlen(cns)+1, mtGC); - strcpy(_name_space, cns); + strcpy(_name_space, cns); // copy cns into _name_space const char* cname = PerfDataManager::counter_name(_name_space, "timestamp"); _timestamp = PerfDataManager::create_long_variable(SUN_GC, cname, PerfData::U_None, CHECK); @@ -59,6 +61,7 @@ ShenandoahHeapRegionCounters::ShenandoahHeapRegionCounters() : PerfData::U_None, CHECK); _regions_data = NEW_C_HEAP_ARRAY(PerfVariable*, num_regions, mtGC); + // Initializing performance data resources for each region for (uint i = 0; i < num_regions; i++) { const char* reg_name = PerfDataManager::name_space(_name_space, "region", i); const char* data_name = PerfDataManager::counter_name(reg_name, "data"); @@ -68,31 +71,33 @@ ShenandoahHeapRegionCounters::ShenandoahHeapRegionCounters() : _regions_data[i] = PerfDataManager::create_long_variable(SUN_GC, data_name, PerfData::U_None, CHECK); } + + if (ShenandoahLogRegionSampling) { + _log_file = new ShenandoahLogFileOutput(ShenandoahRegionSamplingFile, _timestamp->get_value()); + _log_file->initialize(tty); + } } } ShenandoahHeapRegionCounters::~ShenandoahHeapRegionCounters() { if (_name_space != NULL) FREE_C_HEAP_ARRAY(char, _name_space); + if (_log_file != NULL) FREE_C_HEAP_OBJ(_log_file); } void ShenandoahHeapRegionCounters::update() { if (ShenandoahRegionSampling) { jlong current = nanos_to_millis(os::javaTimeNanos()); jlong last = _last_sample_millis; - if (current - last > ShenandoahRegionSamplingRate && - Atomic::cmpxchg(&_last_sample_millis, last, current) == last) { + if (current - last > ShenandoahRegionSamplingRate && Atomic::cmpxchg(&_last_sample_millis, last, current) == last) { ShenandoahHeap* heap = ShenandoahHeap::heap(); - _status->set_value(encode_heap_status(heap)); - _timestamp->set_value(os::elapsed_counter()); - size_t num_regions = heap->num_regions(); - { ShenandoahHeapLocker locker(heap->lock()); size_t rs = ShenandoahHeapRegion::region_size_bytes(); + size_t num_regions = heap->num_regions(); for (uint i = 0; i < num_regions; i++) { ShenandoahHeapRegion* r = heap->get_region(i); jlong data = 0; @@ -108,6 +113,11 @@ void ShenandoahHeapRegionCounters::update() { data |= (r->state_ordinal() & STATUS_MASK) << STATUS_SHIFT; _regions_data[i]->set_value(data); } + + // If logging enabled, dump current region snapshot to log file + if (ShenandoahLogRegionSampling && _log_file != NULL) { + _log_file->write_snapshot(_regions_data, _timestamp, _status, num_regions, rs >> 10); + } } } } @@ -157,7 +167,7 @@ jlong ShenandoahHeapRegionCounters::encode_heap_status(ShenandoahHeap* heap) { status |= (1 << 2); } log_develop_trace(gc)("%s, phase=%u, old_mark=%s, status=" JLONG_FORMAT, - generation->name(), phase, BOOL_TO_STR(heap->is_concurrent_old_mark_in_progress()), status); + generation->name(), phase, BOOL_TO_STR(heap->is_concurrent_old_mark_in_progress()), status); } if (heap->is_degenerated_gc_in_progress()) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.hpp index e559cf67069f3..56ccaecb52b5f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.hpp @@ -26,6 +26,8 @@ #define SHARE_GC_SHENANDOAH_SHENANDOAHHEAPREGIONCOUNTERS_HPP #include "memory/allocation.hpp" +#include "logging/logFileStreamOutput.hpp" +#include "gc/shenandoah/shenandoahLogFileOutput.hpp" /** * This provides the following in JVMStat: @@ -86,6 +88,8 @@ class ShenandoahHeapRegionCounters : public CHeapObj { PerfLongVariable* _status; volatile jlong _last_sample_millis; + uint _count = 0; + ShenandoahLogFileOutput* _log_file; public: ShenandoahHeapRegionCounters(); ~ShenandoahHeapRegionCounters(); @@ -93,6 +97,7 @@ class ShenandoahHeapRegionCounters : public CHeapObj { private: static jlong encode_heap_status(ShenandoahHeap* heap) ; + }; #endif // SHARE_GC_SHENANDOAH_SHENANDOAHHEAPREGIONCOUNTERS_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahLogFileOutput.cpp b/src/hotspot/share/gc/shenandoah/shenandoahLogFileOutput.cpp new file mode 100644 index 0000000000000..2a2c0e279effe --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahLogFileOutput.cpp @@ -0,0 +1,216 @@ +/* + * Copyright (c) 2021, Amazon.com, Inc. All rights reserved. + * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "precompiled.hpp" +#include "jvm.h" +#include "logging/logConfiguration.hpp" +#include "logging/logFileStreamOutput.hpp" +#include "runtime/arguments.hpp" +#include "runtime/os.inline.hpp" +#include "runtime/perfData.inline.hpp" +#include "utilities/globalDefinitions.hpp" +#include "utilities/defaultStream.hpp" + +#include "gc/shenandoah/shenandoahLogFileOutput.hpp" + +const char* const ShenandoahLogFileOutput::Prefix = "file="; +const char* const ShenandoahLogFileOutput::FileOpenMode = "w+"; +const char* const ShenandoahLogFileOutput::PidFilenamePlaceholder = "%p"; +const char* const ShenandoahLogFileOutput::TimestampFilenamePlaceholder = "%t"; +const char* const ShenandoahLogFileOutput::TimestampFormat = "%Y-%m-%d_%H-%M-%S"; +char ShenandoahLogFileOutput::_pid_str[PidBufferSize]; +char ShenandoahLogFileOutput::_vm_start_time_str[StartTimeBufferSize]; + +#define WRITE_LOG_WITH_RESULT_CHECK(op, total) \ +{ \ + int result = op; \ + if (result < 0) { \ + if (!_write_error_is_shown) { \ + jio_fprintf(defaultStream::error_stream(), \ + "Could not write log: %s\n", name()); \ + jio_fprintf(_stream, "\nERROR: Could not write log\n"); \ + _write_error_is_shown = true; \ + return -1; \ + } \ + } \ + total += result; \ +} + +ShenandoahLogFileOutput::ShenandoahLogFileOutput(const char* name, jlong vm_start_time) + : _name(os::strdup_check_oom(name, mtLogging)), _file_name(NULL), _stream(NULL) { + set_file_name_parameters(vm_start_time); + _file_name = make_file_name(name, _pid_str, _vm_start_time_str); +} + +ShenandoahLogFileOutput::~ShenandoahLogFileOutput() { + if (_stream != NULL) { + if (fclose(_stream) != 0) { + jio_fprintf(defaultStream::error_stream(), "Could not close log file '%s' (%s).\n", + _file_name, os::strerror(errno)); + } + } + os::free(_file_name); + os::free(const_cast(_name)); +} + +bool ShenandoahLogFileOutput::flush() { + bool result = true; + if (fflush(_stream) != 0) { + if (!_write_error_is_shown) { + jio_fprintf(defaultStream::error_stream(), + "Could not flush log: %s (%s (%d))\n", name(), os::strerror(errno), errno); + jio_fprintf(_stream, "\nERROR: Could not flush log (%d)\n", errno); + _write_error_is_shown = true; + } + result = false; + } + return result; +} + +void ShenandoahLogFileOutput::initialize(outputStream* errstream) { + _stream = os::fopen(_file_name, ShenandoahLogFileOutput::FileOpenMode); + if (_stream == NULL) { + errstream->print_cr("Error opening log file '%s': %s", _file_name, os::strerror(errno)); + _file_name = make_file_name("./shenandoahSnapshots_pid%p.log", _pid_str, _vm_start_time_str); + _stream = os::fopen(_file_name, ShenandoahLogFileOutput::FileOpenMode); + errstream->print_cr("Writing to default log file: %s", _file_name); + } +} + +int ShenandoahLogFileOutput::write_snapshot(PerfLongVariable** regions, + PerfLongVariable* ts, + PerfLongVariable* status, + size_t num_regions, + size_t rs) { + int written = 0; + FileLocker flocker(_stream); + WRITE_LOG_WITH_RESULT_CHECK(jio_fprintf(_stream, "%lli %lli %u %u\n", + ts->get_value(), + status->get_value(), + num_regions, + rs),written); + if (num_regions > 0) { + WRITE_LOG_WITH_RESULT_CHECK(jio_fprintf(_stream, "%lli", regions[0]->get_value()), written); + } + for (uint i = 1; i < num_regions; ++i) { + WRITE_LOG_WITH_RESULT_CHECK(jio_fprintf(_stream, " %lli", regions[i]->get_value()), written); + } + jio_fprintf(_stream, "\n"); + return flush() ? written : -1; +} + +void ShenandoahLogFileOutput::set_file_name_parameters(jlong vm_start_time) { + int res = jio_snprintf(_pid_str, sizeof(_pid_str), "%d", os::current_process_id()); + assert(res > 0, "PID buffer too small"); + + struct tm local_time; + time_t utc_time = vm_start_time / 1000; + os::localtime_pd(&utc_time, &local_time); + res = (int)strftime(_vm_start_time_str, sizeof(_vm_start_time_str), TimestampFormat, &local_time); + assert(res > 0, "VM start time buffer too small."); +} + +char* ShenandoahLogFileOutput::make_file_name(const char* file_name, + const char* pid_string, + const char* timestamp_string) { + char* result = NULL; + + // Lets start finding out if we have any %d and/or %t in the name. + // We will only replace the first occurrence of any placeholder + const char* pid = strstr(file_name, PidFilenamePlaceholder); + const char* timestamp = strstr(file_name, TimestampFilenamePlaceholder); + + if (pid == NULL && timestamp == NULL) { + // We found no place-holders, return the simple filename + return os::strdup_check_oom(file_name, mtLogging); + } + + // At least one of the place-holders were found in the file_name + const char* first = ""; + size_t first_pos = SIZE_MAX; + size_t first_replace_len = 0; + + const char* second = ""; + size_t second_pos = SIZE_MAX; + size_t second_replace_len = 0; + + // If we found a %p, then setup our variables accordingly + if (pid != NULL) { + if (timestamp == NULL || pid < timestamp) { + first = pid_string; + first_pos = pid - file_name; + first_replace_len = strlen(PidFilenamePlaceholder); + } else { + second = pid_string; + second_pos = pid - file_name; + second_replace_len = strlen(PidFilenamePlaceholder); + } + } + + if (timestamp != NULL) { + if (pid == NULL || timestamp < pid) { + first = timestamp_string; + first_pos = timestamp - file_name; + first_replace_len = strlen(TimestampFilenamePlaceholder); + } else { + second = timestamp_string; + second_pos = timestamp - file_name; + second_replace_len = strlen(TimestampFilenamePlaceholder); + } + } + + size_t first_len = strlen(first); + size_t second_len = strlen(second); + + // Allocate the new buffer, size it to hold all we want to put in there +1. + size_t result_len = strlen(file_name) + first_len - first_replace_len + second_len - second_replace_len; + result = NEW_C_HEAP_ARRAY(char, result_len + 1, mtLogging); + + // Assemble the strings + size_t file_name_pos = 0; + size_t i = 0; + while (i < result_len) { + if (file_name_pos == first_pos) { + // We are in the range of the first placeholder + strcpy(result + i, first); + // Bump output buffer position with length of replacing string + i += first_len; + // Bump source buffer position to skip placeholder + file_name_pos += first_replace_len; + } else if (file_name_pos == second_pos) { + // We are in the range of the second placeholder + strcpy(result + i, second); + i += second_len; + file_name_pos += second_replace_len; + } else { + // Else, copy char by char of the original file + result[i] = file_name[file_name_pos++]; + i++; + } + } + // Add terminating char + result[result_len] = '\0'; + return result; +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahLogFileOutput.hpp b/src/hotspot/share/gc/shenandoah/shenandoahLogFileOutput.hpp new file mode 100644 index 0000000000000..9a3b8a880d563 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahLogFileOutput.hpp @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2021, Amazon.com, Inc. All rights reserved. + * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHLOGFILEOUTPUT_HPP +#define SHARE_GC_SHENANDOAH_SHENANDOAHLOGFILEOUTPUT_HPP + +#include "logging/logFileStreamOutput.hpp" +#include "logging/logFileOutput.hpp" +#include "utilities/globalDefinitions.hpp" +#include "runtime/perfData.inline.hpp" + +// Log file output to capture Shenandoah GC data. + +class ShenandoahLogFileOutput : public CHeapObj { +private: + static const char* const FileOpenMode; + static const char* const PidFilenamePlaceholder; + static const char* const TimestampFilenamePlaceholder; + static const char* const TimestampFormat; + static const size_t StartTimeBufferSize = 20; + static const size_t PidBufferSize = 21; + static char _pid_str[PidBufferSize]; + static char _vm_start_time_str[StartTimeBufferSize]; + + const char* _name; + char* _file_name; + FILE* _stream; + + bool _write_error_is_shown; + + bool parse_options(const char* options, outputStream* errstream); + char *make_file_name(const char* file_name, const char* pid_string, const char* timestamp_string); + + bool flush(); + +public: + ShenandoahLogFileOutput(const char *name, jlong vm_start_time); + ~ShenandoahLogFileOutput(); + + void initialize(outputStream* errstream); + + int write_snapshot(PerfLongVariable** regions, + PerfLongVariable* ts, + PerfLongVariable* status, + size_t num_regions, + size_t rs); + + const char* name() const { + return _name; + } + + static const char* const Prefix; + static void set_file_name_parameters(jlong start_time); +}; +#endif //SHARE_GC_SHENANDOAH_SHENANDOAHLOGFILEOUTPUT_HPP + diff --git a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp index 3ff9e6278550f..47066cbab5320 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp @@ -207,6 +207,15 @@ "the samples. Higher values provide more fidelity, at expense " \ "of more sampling overhead.") \ \ + product(bool, ShenandoahLogRegionSampling, false, \ + "Save region sampling stream to ShenandoahRegionSamplingFile") \ + \ + product(ccstr, ShenandoahRegionSamplingFile, \ + "./shenandoahSnapshots_pid%p.log", \ + "If ShenandoahLogRegionSampling is on, save sampling data stream "\ + "to this file [default: ./shenandoahSnapshots_pid%p.log] " \ + "(%p replaced with pid)") \ + \ product(uintx, ShenandoahControlIntervalMin, 1, EXPERIMENTAL, \ "The minimum sleep interval for the control loop that drives " \ "the cycles. Lower values would increase GC responsiveness " \ @@ -432,8 +441,7 @@ product(uintx, ShenandoahAgingCyclePeriod, 1, EXPERIMENTAL, \ "With generational mode, increment the age of objects and" \ "regions each time this many young-gen GC cycles are completed.") - -// end of GC_SHENANDOAH_FLAGS + // end of GC_SHENANDOAH_FLAGS // 2^ShenandoahTenuredRegionUsageBiasLogBase2 is 128 #define ShenandoahTenuredRegionUsageBiasLogBase2 7 diff --git a/src/hotspot/share/logging/logFileStreamOutput.cpp b/src/hotspot/share/logging/logFileStreamOutput.cpp index 60296e4f05080..d1f0b4483d7d5 100644 --- a/src/hotspot/share/logging/logFileStreamOutput.cpp +++ b/src/hotspot/share/logging/logFileStreamOutput.cpp @@ -76,20 +76,6 @@ int LogFileStreamOutput::write_decorations(const LogDecorations& decorations) { return total_written; } -class FileLocker : public StackObj { -private: - FILE *_file; - -public: - FileLocker(FILE *file) : _file(file) { - os::flockfile(_file); - } - - ~FileLocker() { - os::funlockfile(_file); - } -}; - bool LogFileStreamOutput::flush() { bool result = true; if (fflush(_stream) != 0) { diff --git a/src/hotspot/share/logging/logFileStreamOutput.hpp b/src/hotspot/share/logging/logFileStreamOutput.hpp index d211e8bda2a13..fe84a3a236c24 100644 --- a/src/hotspot/share/logging/logFileStreamOutput.hpp +++ b/src/hotspot/share/logging/logFileStreamOutput.hpp @@ -98,4 +98,18 @@ class LogStderrOutput : public LogFileStreamOutput { extern LogStderrOutput &StderrLog; extern LogStdoutOutput &StdoutLog; +class FileLocker : public StackObj { +private: + FILE *_file; + +public: + FileLocker(FILE *file) : _file(file) { + os::flockfile(_file); + } + + ~FileLocker() { + os::funlockfile(_file); + } +}; + #endif // SHARE_LOGGING_LOGFILESTREAMOUTPUT_HPP diff --git a/test/hotspot/jtreg/gc/shenandoah/TestRegionSamplingLogging.java b/test/hotspot/jtreg/gc/shenandoah/TestRegionSamplingLogging.java new file mode 100644 index 0000000000000..74a2f641d298e --- /dev/null +++ b/test/hotspot/jtreg/gc/shenandoah/TestRegionSamplingLogging.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2017, 2018, Red Hat, Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +/* + * @test id=adaptive + * @requires vm.gc.Shenandoah + * + * @run main/othervm -Xmx1g -Xms1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * -XX:+ShenandoahRegionSampling -XX:+ShenandoahRegionSampling + * -XX:+ShenandoahLogRegionSampling -XX:ShenandoahRegionSamplingFile=region-snapshots-%p.log + * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive + * TestRegionSamplingLogging + */ + +import java.io.File; +import java.util.Arrays; + +public class TestRegionSamplingLogging { + + static final long TARGET_MB = Long.getLong("target", 2_000); // 2 Gb allocation + + static volatile Object sink; + + public static void main(String[] args) throws Exception { + long count = TARGET_MB * 1024 * 1024 / 16; + for (long c = 0; c < count; c++) { + sink = new Object(); + } + + File directory = new File("."); + File[] files = directory.listFiles((dir, name) -> name.startsWith("region-snapshots") && name.endsWith(".log")); + System.out.println(Arrays.toString(files)); + if (files == null || files.length == 0) { + throw new IllegalStateException("Did not find expected snapshot log file."); + } + } +} From 7765c14461f3343c067a2de805a55d83d4b63835 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Thu, 7 Oct 2021 21:16:27 +0000 Subject: [PATCH 078/254] Inline usage of pointer_delta Reviewed-by: rkennke --- src/hotspot/share/gc/shared/cardTable.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/shared/cardTable.hpp b/src/hotspot/share/gc/shared/cardTable.hpp index 94f5265112596..dc9246a9223f4 100644 --- a/src/hotspot/share/gc/shared/cardTable.hpp +++ b/src/hotspot/share/gc/shared/cardTable.hpp @@ -188,7 +188,7 @@ class CardTable: public CHeapObj { "out of bounds access to card marking array. p: " PTR_FORMAT " _byte_map: " PTR_FORMAT " _byte_map + _byte_map_size: " PTR_FORMAT, p2i(p), p2i(_byte_map), p2i(_byte_map + _byte_map_size)); - size_t delta = pointer_delta(p, _byte_map_base, sizeof(CardValue)); + size_t delta = (((uintptr_t) p) - ((uintptr_t) _byte_map_base)) / sizeof(CardValue); HeapWord* result = (HeapWord*) (delta << card_shift); assert(_whole_heap.contains(result), "Returning result = " PTR_FORMAT " out of bounds of " From fb788c84ca55d3ddce0af21109cff6f08931e1dd Mon Sep 17 00:00:00 2001 From: William Kemper Date: Mon, 11 Oct 2021 22:30:31 +0000 Subject: [PATCH 079/254] Register filler objects when non-lab allocations are backed out Reviewed-by: kdnilsen --- .../gc/shenandoah/shenandoahHeap.inline.hpp | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp index 6fab11347f01a..036f45d87f1b4 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp @@ -418,16 +418,17 @@ inline oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, Shenandoah oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val); if (result == copy_val) { // Successfully evacuated. Our copy is now the public one! - if (target_gen == OLD_GENERATION) { - handle_old_evacuation(copy, size, from_region->is_young()); - } else if (target_gen == YOUNG_GENERATION) { - if (is_aging_cycle()) { - ShenandoahHeap::increase_object_age(copy_val, from_region->age() + 1); + if (mode()->is_generational()) { + if (target_gen == OLD_GENERATION) { + handle_old_evacuation(copy, size, from_region->is_young()); + } else if (target_gen == YOUNG_GENERATION) { + if (is_aging_cycle()) { + ShenandoahHeap::increase_object_age(copy_val, from_region->age() + 1); + } + } else { + ShouldNotReachHere(); } - } else { - ShouldNotReachHere(); } - shenandoah_assert_correct(NULL, copy_val); return copy_val; } else { @@ -460,7 +461,9 @@ inline oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, Shenandoah // we have to keep the fwdptr initialized and pointing to our (stale) copy. fill_with_object(copy, size); shenandoah_assert_correct(NULL, copy_val); - // For non-LAB allocations, the object has already been registered + if (mode()->is_generational() && target_gen == OLD_GENERATION) { + card_scan()->register_object(copy); + } } shenandoah_assert_correct(NULL, result); return result; From deb452d749339f21ecef18988bbb68b51def2fb2 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Tue, 12 Oct 2021 16:03:25 +0000 Subject: [PATCH 080/254] Update heap usage tallies for generations after full GC compaction Reviewed-by: rkennke --- .../share/gc/shenandoah/shenandoahFullGC.cpp | 98 ++++--------------- .../gc/shenandoah/shenandoahVerifier.cpp | 60 +++++++++--- 2 files changed, 66 insertions(+), 92 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index d1b98be783e71..345fa691efad0 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -377,12 +377,10 @@ class ShenandoahPrepareForCompactionTask : public AbstractGangTask { ShenandoahHeap* const _heap; ShenandoahHeapRegionSet** const _worker_slices; size_t const _num_workers; - size_t *_old_used; - size_t *_young_used; public: ShenandoahPrepareForCompactionTask(PreservedMarksSet *preserved_marks, ShenandoahHeapRegionSet **worker_slices, - size_t num_workers, size_t* old_used, size_t* young_used); + size_t num_workers); static bool is_candidate_region(ShenandoahHeapRegion* r) { // Empty region: get it into the slice to defragment the slice itself. @@ -397,10 +395,6 @@ class ShenandoahPrepareForCompactionTask : public AbstractGangTask { } void work(uint worker_id); - void add_to_young_used(size_t worker_id, size_t amount); - void add_to_old_used(size_t worker_id, size_t amount); - size_t young_used(); - size_t old_used(); }; class ShenandoahPrepareForGenerationalCompactionObjectClosure : public ObjectClosure { @@ -464,7 +458,6 @@ class ShenandoahPrepareForGenerationalCompactionObjectClosure : public ObjectClo if (_old_to_region != nullptr) { log_debug(gc)("Planned compaction into Old Region " SIZE_FORMAT ", used: " SIZE_FORMAT " tabulated by worker %u", _old_to_region->index(), _old_compact_point - _old_to_region->bottom(), _worker_id); - _compactor->add_to_old_used(_worker_id, _old_compact_point - _old_to_region->bottom()); _old_to_region->set_new_top(_old_compact_point); _old_to_region = nullptr; } @@ -474,7 +467,6 @@ class ShenandoahPrepareForGenerationalCompactionObjectClosure : public ObjectClo if (_young_to_region != nullptr) { log_debug(gc)("Worker %u planned compaction into Young Region " SIZE_FORMAT ", used: " SIZE_FORMAT, _worker_id, _young_to_region->index(), _young_compact_point - _young_to_region->bottom()); - _compactor->add_to_young_used(_worker_id, _young_compact_point - _young_to_region->bottom()); _young_to_region->set_new_top(_young_compact_point); _young_to_region = nullptr; } @@ -650,16 +642,10 @@ class ShenandoahPrepareForCompactionObjectClosure : public ObjectClosure { ShenandoahPrepareForCompactionTask::ShenandoahPrepareForCompactionTask(PreservedMarksSet *preserved_marks, ShenandoahHeapRegionSet **worker_slices, - size_t num_workers, size_t* old_used, size_t* young_used) : + size_t num_workers) : AbstractGangTask("Shenandoah Prepare For Compaction"), _preserved_marks(preserved_marks), _heap(ShenandoahHeap::heap()), - _worker_slices(worker_slices), _num_workers(num_workers), - _old_used(old_used), _young_used(young_used) { - for (size_t i = 0; i < _num_workers; i++) { - _old_used[i] = 0; - _young_used[i] = 0; - } -} + _worker_slices(worker_slices), _num_workers(num_workers) { } void ShenandoahPrepareForCompactionTask::work(uint worker_id) { @@ -731,44 +717,6 @@ void ShenandoahPrepareForCompactionTask::work(uint worker_id) { } } -// Accumulate HeapWords of memory used in young-gen memory. -void ShenandoahPrepareForCompactionTask::add_to_young_used(size_t worker_id, size_t amount) { - log_debug(gc)("Adding to _young_used for worker_id: " SIZE_FORMAT ", amount: " SIZE_FORMAT, worker_id, amount); - _young_used[worker_id] += amount; -} - -// Accumulate HeapWords of memory used in old-gen memory. -void ShenandoahPrepareForCompactionTask::add_to_old_used(size_t worker_id, size_t amount) { - log_debug(gc)("Adding to _old_used for worker_id: " SIZE_FORMAT ", amount: " SIZE_FORMAT, worker_id, amount); - _old_used[worker_id] += amount; -} - -// Return total number of bytes used in young-gen memory -size_t ShenandoahPrepareForCompactionTask::young_used() { - size_t result = 0; - log_debug(gc)("Calculating young_used by accumulating worker totals"); - for (size_t i = 0; i < _num_workers; i++) { - log_debug(gc)(" worker [" SIZE_FORMAT "] contributed " SIZE_FORMAT, i, _young_used[i]); - result += _young_used[i]; - } - result *= HeapWordSize; - log_debug(gc)("Accumulated _young_used is: " SIZE_FORMAT, result); - return result; -} - -// Return total number of bytes used in old-gen memory -size_t ShenandoahPrepareForCompactionTask::old_used() { - size_t result = 0; - log_debug(gc)("Calculating old_used by accumulating worker totals"); - for (size_t i = 0; i < _num_workers; i++) { - log_debug(gc)(" worker [" SIZE_FORMAT "] contributed " SIZE_FORMAT, i, _old_used[i]); - result += _old_used[i]; - } - log_debug(gc)("Accumulated _old_used is: " SIZE_FORMAT, result); - result *= HeapWordSize; - return result; -} - void ShenandoahFullGC::calculate_target_humongous_objects() { ShenandoahHeap* heap = ShenandoahHeap::heap(); @@ -789,17 +737,6 @@ void ShenandoahFullGC::calculate_target_humongous_objects() { log_debug(gc)("Full GC calculating target humongous objects from end " SIZE_FORMAT, to_end); for (size_t c = heap->num_regions(); c > 0; c--) { ShenandoahHeapRegion *r = heap->get_region(c - 1); - - if (r->is_humongous_start() && heap->mode()->is_generational()) { - oop obj = cast_to_oop(r->bottom()); - size_t humongous_bytes = obj->size() * HeapWordSize; - log_debug(gc)("Adjusting used for humongous %s object by " SIZE_FORMAT, r->is_old()? "OLD": "YOUNG", humongous_bytes); - if (r->is_old()) { - heap->old_generation()->increase_used(humongous_bytes); - } else { - heap->young_generation()->increase_used(humongous_bytes); - } - } if (r->is_humongous_continuation() || (r->new_top() == r->bottom())) { // To-region candidate: record this, and continue scan to_begin = r->index(); @@ -1070,21 +1007,8 @@ void ShenandoahFullGC::phase2_calculate_target_addresses(ShenandoahHeapRegionSet size_t num_workers = heap->max_workers(); ResourceMark rm; - size_t* old_used = NEW_RESOURCE_ARRAY(size_t, num_workers); - size_t* young_used = NEW_RESOURCE_ARRAY(size_t, num_workers); - - ShenandoahPrepareForCompactionTask task(_preserved_marks, worker_slices, num_workers, old_used, young_used); + ShenandoahPrepareForCompactionTask task(_preserved_marks, worker_slices, num_workers); heap->workers()->run_task(&task); - - if (heap->mode()->is_generational()) { - log_debug(gc)("Usage after compacting regular objects is young: " SIZE_FORMAT ", old: " SIZE_FORMAT, - task.young_used(), task.old_used()); - heap->young_generation()->increase_used(task.young_used()); - heap->old_generation()->increase_used(task.old_used()); - } - - FREE_RESOURCE_ARRAY(size_t, old_used, num_workers); - FREE_RESOURCE_ARRAY(size_t, young_used, num_workers); } // Compute the new addresses for humongous objects @@ -1293,6 +1217,15 @@ class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure { r->recycle(); } + // Update final usage for generations + if (_heap->mode()->is_generational() && live != 0) { + if (r->is_young()) { + _heap->young_generation()->increase_used(live); + } else if (r->is_old()) { + _heap->old_generation()->increase_used(live); + } + } + r->set_live_data(live); r->reset_alloc_metadata(); _live += live; @@ -1432,6 +1365,11 @@ void ShenandoahFullGC::phase4_compact_objects(ShenandoahHeapRegionSet** worker_s { ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_rebuild); + if (heap->mode()->is_generational()) { + heap->young_generation()->clear_used(); + heap->old_generation()->clear_used(); + } + ShenandoahPostCompactClosure post_compact; heap->heap_region_iterate(&post_compact); heap->set_used(post_compact.get_live()); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp index 1ead9cb228320..c952e9758807d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp @@ -358,6 +358,44 @@ class ShenandoahCalculateRegionStatsClosure : public ShenandoahHeapRegionClosure size_t garbage() { return _garbage; } }; +class ShenandoahGenerationStatsClosure : public ShenandoahHeapRegionClosure { + public: + ShenandoahCalculateRegionStatsClosure old; + ShenandoahCalculateRegionStatsClosure young; + ShenandoahCalculateRegionStatsClosure global; + + void heap_region_do(ShenandoahHeapRegion* r) override { + switch (r->affiliation()) { + default: + ShouldNotReachHere(); + return; + case FREE: return; + case YOUNG_GENERATION: + young.heap_region_do(r); + break; + case OLD_GENERATION: + old.heap_region_do(r); + break; + } + global.heap_region_do(r); + } + + static void log_usage(ShenandoahGeneration* generation, ShenandoahCalculateRegionStatsClosure& stats) { + log_debug(gc)("Safepoint verification: %s verified usage: " SIZE_FORMAT "%s, recorded usage: " SIZE_FORMAT "%s", + generation->name(), + byte_size_in_proper_unit(generation->used()), proper_unit_for_byte_size(generation->used()), + byte_size_in_proper_unit(stats.used()), proper_unit_for_byte_size(stats.used())); + } + + static void validate_usage(const char* label, ShenandoahGeneration* generation, ShenandoahCalculateRegionStatsClosure& stats) { + guarantee(stats.used() == generation->used(), + "%s: generation (%s) used size must be consistent: generation-used = " SIZE_FORMAT "%s, regions-used = " SIZE_FORMAT "%s", + label, generation->name(), + byte_size_in_proper_unit(generation->used()), proper_unit_for_byte_size(generation->used()), + byte_size_in_proper_unit(stats.used()), proper_unit_for_byte_size(stats.used())); + } +}; + class ShenandoahVerifyHeapRegionClosure : public ShenandoahHeapRegionClosure { private: ShenandoahHeap* _heap; @@ -779,20 +817,18 @@ void ShenandoahVerifier::verify_at_safepoint(const char* label, _heap->verify_rem_set_after_full_gc(); } - ShenandoahCalculateRegionStatsClosure cl; - generation->heap_region_iterate(&cl); - - log_debug(gc)("Safepoint verification: generation %s usage hereby calculated as: " SIZE_FORMAT, - generation->name(), cl.used()); - log_debug(gc)(" previous tabulation of usage: " SIZE_FORMAT, generation->used()); + ShenandoahGenerationStatsClosure cl; + _heap->heap_region_iterate(&cl); + if (LogTarget(Debug, gc)::is_enabled()) { + ShenandoahGenerationStatsClosure::log_usage(_heap->old_generation(), cl.old); + ShenandoahGenerationStatsClosure::log_usage(_heap->young_generation(), cl.young); + ShenandoahGenerationStatsClosure::log_usage(_heap->global_generation(), cl.global); + } - size_t generation_used = generation->used(); - guarantee(cl.used() == generation_used, - "%s: generation (%s) used size must be consistent: generation-used = " SIZE_FORMAT "%s, regions-used = " SIZE_FORMAT "%s", - label, generation->name(), - byte_size_in_proper_unit(generation_used), proper_unit_for_byte_size(generation_used), - byte_size_in_proper_unit(cl.used()), proper_unit_for_byte_size(cl.used())); + ShenandoahGenerationStatsClosure::validate_usage(label, _heap->old_generation(), cl.old); + ShenandoahGenerationStatsClosure::validate_usage(label, _heap->young_generation(), cl.young); + ShenandoahGenerationStatsClosure::validate_usage(label, _heap->global_generation(), cl.global); } log_debug(gc)("Safepoint verification finished remembered set verification"); From 5ccbfbc3f77bc6b81d5a5cf9edd65c78a52e603b Mon Sep 17 00:00:00 2001 From: William Kemper Date: Tue, 12 Oct 2021 16:04:12 +0000 Subject: [PATCH 081/254] Remove vestigial command line options and removed unused code Reviewed-by: rkennke --- .../share/gc/shenandoah/shenandoahHeap.cpp | 25 +--- .../shenandoah/shenandoahScanRemembered.hpp | 28 +---- .../shenandoahScanRemembered.inline.hpp | 112 +----------------- .../gc/shenandoah/shenandoah_globals.hpp | 8 -- 4 files changed, 4 insertions(+), 169 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 282a689a80017..087fb645f5909 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -2301,11 +2301,7 @@ class ShenandoahUpdateHeapRefsTask : public AbstractGangTask { _heap->marked_object_oop_iterate(r, &cl, update_watermark); } else { // Old region in a young cycle or mixed cycle. - if (ShenandoahUseSimpleCardScanning) { - if (ShenandoahBarrierSet::barrier_set()->card_table()->is_dirty(MemRegion(r->bottom(), r->top()))) { - update_all_references(&cl, r, update_watermark ); - } - } else if (!_mixed_evac) { + if (!_mixed_evac) { // This is a young evac.. _heap->card_scan()->process_region(r, &cl, true); } else { @@ -2375,25 +2371,6 @@ class ShenandoahUpdateHeapRefsTask : public AbstractGangTask { r = _regions->next(); } } - - template - void update_all_references(T* cl, ShenandoahHeapRegion* r, HeapWord* update_watermark) { - if (r->is_humongous()) { - r->oop_iterate_humongous(cl); - } else { - // We don't have liveness information about this region. - // Therefore we process all objects, rather than just marked ones. - // Otherwise subsequent traversals will encounter stale pointers. - HeapWord* p = r->bottom(); - ShenandoahObjectToOopBoundedClosure objs(cl, p, update_watermark); - // Anything beyond update_watermark is not yet allocated or initialized. - while (p < update_watermark) { - oop obj = cast_to_oop(p); - objs.do_object(obj); - p += obj->size(); - } - } - } }; void ShenandoahHeap::update_heap_references(bool concurrent) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp index 1e0aabcd072d0..bd946a8408cc7 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp @@ -26,10 +26,6 @@ #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHSCANREMEMBERED_HPP #define SHARE_GC_SHENANDOAH_SHENANDOAHSCANREMEMBERED_HPP -// During development of this new feature, we want the option to test -// with and without, and to compare performance before and after. -#define FAST_REMEMBERED_SET_SCANNING - // Terminology used within this source file: // // Card Entry: This is the information that identifies whether a @@ -503,7 +499,6 @@ class ShenandoahCardCluster: public CHeapObj { public: static const size_t CardsPerCluster = 64; -#ifdef FAST_REMEMBERED_SET_SCANNING private: // This bit is set iff at least one object starts within a // particular card region. @@ -513,8 +508,6 @@ class ShenandoahCardCluster: public CHeapObj { static const uint16_t FirstStartShift = 0; static const uint16_t LastStartShift = 6; - static const uint16_t CardOffsetMultiplier = 8; - uint16_t *object_starts; public: @@ -542,19 +535,16 @@ class ShenandoahCardCluster: public CHeapObj { while (card_index <= last_card_index) object_starts[card_index++] = 0; } -#endif // FAST_REMEMBERED_SET_SCANNING ShenandoahCardCluster(RememberedSet *rs) { _rs = rs; -#ifdef FAST_REMEMBERED_SET_SCANNING - // HEY! We don't really need object_starts entries for every card entry. We only need these for the + // HEY! We don't really need object_starts entries for every card entry. We only need these for // the card entries that correspond to old-gen memory. But for now, let's be quick and dirty. object_starts = (uint16_t *) malloc(rs->total_cards() * sizeof(uint16_t)); if (object_starts == NULL) fatal("Insufficient memory for initializing heap"); for (size_t i = 0; i < rs->total_cards(); i++) object_starts[i] = 0; -#endif } ~ShenandoahCardCluster() { @@ -643,24 +633,8 @@ class ShenandoahCardCluster: public CHeapObj { // time, so we get a lot of benefit from each investment // in registering an object. -private: - const size_t CardByteOffsetMultiplier = 8; - const size_t CardWordOffsetMultiplier = 1; - -public: -#ifdef CROSSING_OFFSETS_NO_LONGER_NEEDED -private: - const uint16_t CrossingObjectOverflow = 0x7fff; - public: - // This has side effect of clearing ObjectStartsInCardRegion bit. - inline void set_crossing_object_start(size_t card_index, uint16_t crossing_offset) { - object_starts[card_index] = crossing_offset; - } -#endif // CROSSING_OFFSETS_NO_LONGER_NEEDED - - // The starting locations of objects contained within old-gen memory // are registered as part of the remembered set implementation. This // information is required when scanning dirty card regions that are diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp index d350d6ae05ddd..f033c4b16847a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp @@ -206,7 +206,7 @@ ShenandoahCardCluster::register_object_wo_lock(HeapWord* address) template inline void ShenandoahCardCluster::coalesce_objects(HeapWord* address, size_t length_in_words) { -#ifdef FAST_REMEMBERED_SET_SCANNING + size_t card_at_start = _rs->card_index_for_addr(address); HeapWord *card_start_address = _rs->addr_for_card_index(card_at_start); size_t card_at_end = card_at_start + ((address + length_in_words) - card_start_address) / CardTable::card_size_in_words; @@ -246,136 +246,28 @@ ShenandoahCardCluster::coalesce_objects(HeapWord* address, size_t // card_at_end had an object that starts after the coalesced object, so no changes required for card_at_end } -#else // FAST_REMEMBERED_SET_SCANNING - // Do nothing for now as we have a brute-force implementation - // of findSpanningObject(). -#endif // FAST_REMEMBERED_SET_SCANNING } template inline bool ShenandoahCardCluster::has_object(size_t card_index) { -#ifdef FAST_REMEMBERED_SET_SCANNING - if (object_starts[card_index] & ObjectStartsInCardRegion) - return true; - else - return false; -#else // FAST_REMEMBERED_SET_SCANNING' - ShenandoahHeap *heap = ShenandoahHeap::heap(); - HeapWord *addr = _rs->addr_for_card_index(card_index); - ShenandoahHeapRegion *region = heap->heap_region_containing(addr); - - // region->block_start(addr) is not robust to inquiries beyond top() and it crashes. - if (region->top() <= addr) - return false; - - // region->block_start(addr) is also not robust to inquiries within a humongous continuation region. - // if region is humongous continuation, no object starts within it. - if (region->is_humongous_continuation()) - return false; - - HeapWord *obj = region->block_start(addr); - - // addr is the first address of the card region. - // obj is the object that spans addr (or starts at addr). - assert(obj != NULL, "Object cannot be null"); - if (obj >= addr) - return true; - else { - HeapWord *end_addr = addr + CardTable::card_size_in_words; - - // end_addr needs to be adjusted downward if top address of the enclosing region is less than end_addr. this is intended - // to be slow and reliable alternative to the planned production quality replacement, so go ahead and spend some extra - // cycles here in order to make this code reliable. - if (region->top() < end_addr) { - end_addr = region->top(); - } - - obj += oop(obj)->size(); - if (obj < end_addr) - return true; - else - return false; - } -#endif // FAST_REMEMBERED_SET_SCANNING' + return object_starts[card_index] & ObjectStartsInCardRegion; } template inline size_t ShenandoahCardCluster::get_first_start(size_t card_index) { -#ifdef FAST_REMEMBERED_SET_SCANNING assert(object_starts[card_index] & ObjectStartsInCardRegion, "Can't get first start because no object starts here"); return (object_starts[card_index] & FirstStartBits) >> FirstStartShift; -#else // FAST_REMEMBERED_SET_SCANNING - HeapWord *addr = _rs->addr_for_card_index(card_index); - ShenandoahHeap *heap = ShenandoahHeap::heap(); - ShenandoahHeapRegion *region = heap->heap_region_containing(addr); - - HeapWord *obj = region->block_start(addr); - - assert(obj != NULL, "Object cannot be null."); - if (obj >= addr) - return obj - addr; - else { - HeapWord *end_addr = addr + CardTable::card_size_in_words; - obj += oop(obj)->size(); - - // If obj > end_addr, offset will reach beyond end of this card - // region. But clients should not invoke this service unless - // they first confirm that this card has an object. - assert(obj < end_addr, "Object out of range"); - return obj - addr; - } -#endif // FAST_REMEMBERED_SET_SCANNING } template inline size_t ShenandoahCardCluster::get_last_start(size_t card_index) { -#ifdef FAST_REMEMBERED_SET_SCANNING assert(object_starts[card_index] & ObjectStartsInCardRegion, "Can't get last start because no objects starts here"); return (object_starts[card_index] & LastStartBits) >> LastStartShift; -#else // FAST_REMEMBERED_SET_SCANNING - HeapWord *addr = _rs->addr_for_card_index(card_index); - HeapWord *end_addr = addr + CardTable::card_size_in_words; - ShenandoahHeap *heap = ShenandoahHeap::heap(); - ShenandoahHeapRegion *region = heap->heap_region_containing(addr); - HeapWord *obj = region->block_start(addr); - assert(obj != NULL, "Object cannot be null."); - - if (region->top() <= end_addr) { - end_addr = region->top(); - } - - HeapWord *end_obj = obj + oop(obj)->size(); - while (end_obj < end_addr) { - obj = end_obj; - end_obj = obj + oop(obj)->size(); - } - assert(obj >= addr, "Object out of range."); - return obj - addr; -#endif // FAST_REMEMBERED_SET_SCANNING -} - -#ifdef CROSSING_OFFSETS_NO_LONGER_NEEDED -template -inline size_t -ShenandoahCardCluster::get_crossing_object_start(size_t card_index) { - HeapWord *addr = _rs->addr_for_card_index(card_index); - size_t cluster_no = card_index / ShenandoahCardCluster::CardsPerCluster; - HeapWord *cluster_addr = _rs->addr_for_card_index(cluster_no * CardsPerCluster); - - ShenandoahHeap *heap = ShenandoahHeap::heap(); - ShenandoahHeapRegion *region = heap->heap_region_containing(addr); - HeapWord *obj = region->block_start(addr); - - if (obj > cluster_addr) - return obj - cluster_addr; - else - return 0x7fff; } -#endif template inline size_t diff --git a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp index 47066cbab5320..55de158f5bbeb 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp @@ -405,10 +405,6 @@ "Fix references with load reference barrier. Disabling this " \ "might degrade performance.") \ \ - product(bool, ShenandoahUseSimpleCardScanning, false, DIAGNOSTIC, \ - "Testing: use simplified, very inefficient but much less complex" \ - " card table scanning.") \ - \ product(uintx, ShenandoahTenuredRegionUsageBias, 192, EXPERIMENTAL, \ "The collection set is comprised of heap regions that contain " \ "the greatest amount of garbage. " \ @@ -430,10 +426,6 @@ "Turn on/off evacuating individual tenured young objects " \ " to the old generation.") \ \ - product(bool, ShenandoahPromoteTenuredRegions, true, DIAGNOSTIC, \ - "Turn on/off transitioning tenured young regions " \ - " to the old generation.") \ - \ product(bool, ShenandoahAllowOldMarkingPreemption, true, DIAGNOSTIC, \ "Allow young generation collections to suspend concurrent" \ " marking in the old generation.") \ From aa775e9971207c4325ef447767940dcdfc11e135 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Wed, 13 Oct 2021 19:03:07 +0000 Subject: [PATCH 082/254] Preempt old preparation Reviewed-by: wkemper --- .../heuristics/shenandoahOldHeuristics.cpp | 1 + .../gc/shenandoah/shenandoahControlThread.cpp | 18 ++- .../gc/shenandoah/shenandoahDegeneratedGC.cpp | 1 - .../share/gc/shenandoah/shenandoahFreeSet.cpp | 8 + .../share/gc/shenandoah/shenandoahHeap.cpp | 11 ++ .../share/gc/shenandoah/shenandoahHeap.hpp | 3 + .../gc/shenandoah/shenandoahHeapRegion.cpp | 24 ++- .../gc/shenandoah/shenandoahHeapRegion.hpp | 26 ++- .../shenandoahHeapRegionCounters.hpp | 1 - .../share/gc/shenandoah/shenandoahOldGC.cpp | 153 +++++++++++------- .../share/gc/shenandoah/shenandoahOldGC.hpp | 4 +- .../shenandoah/shenandoahYoungGeneration.cpp | 6 +- 12 files changed, 181 insertions(+), 75 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp index ba566a76f1988..c8959fb2e0738 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp @@ -181,6 +181,7 @@ void ShenandoahOldHeuristics::prepare_for_old_collections() { total_garbage += garbage; if (region->is_regular()) { + region->reset_coalesce_and_fill_boundary(); if (!region->has_live()) { region->make_trash_immediate(); immediate_regions++; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index 2e88f0fef789f..22ed22c03198d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -208,9 +208,9 @@ void ShenandoahControlThread::run_service() { } else { heap->set_unload_classes(false); } - } else if (heap->is_concurrent_old_mark_in_progress()) { - // Nobody asked us to do anything, but we have an old generation mark - // in progress, so resume working on that. + } else if (heap->is_concurrent_old_mark_in_progress() || heap->is_concurrent_prep_for_mixed_evacuation_in_progress()) { + // Nobody asked us to do anything, but we have an old-generation mark or old-generation preparation for + // mixed evacuation in progress, so resume working on that. cause = GCCause::_shenandoah_concurrent_gc; generation = OLD; set_gc_mode(marking_old); @@ -507,7 +507,10 @@ bool ShenandoahControlThread::check_soft_max_changed() const { } void ShenandoahControlThread::resume_concurrent_old_cycle(ShenandoahGeneration* generation, GCCause::Cause cause) { - assert(ShenandoahHeap::heap()->is_concurrent_old_mark_in_progress(), "Old mark should be in progress"); + + assert(ShenandoahHeap::heap()->is_concurrent_old_mark_in_progress() || + ShenandoahHeap::heap()->is_concurrent_prep_for_mixed_evacuation_in_progress(), + "Old mark or mixed-evac prep should be in progress"); log_debug(gc)("Resuming old generation with " UINT32_FORMAT " marking tasks queued.", generation->task_queues()->tasks()); ShenandoahHeap* heap = ShenandoahHeap::heap(); @@ -516,10 +519,9 @@ void ShenandoahControlThread::resume_concurrent_old_cycle(ShenandoahGeneration* ShenandoahGCSession session(cause, generation); TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); - - // We can only really tolerate being cancelled during concurrent - // marking. This flag here (passed by reference) is used to control - // precisely where the regulator is allowed to cancel a GC. + // We can only tolerate being cancelled during concurrent marking or during preparation for mixed + // evacuation. This flag here (passed by reference) is used to control precisely where the regulator + // is allowed to cancel a GC. ShenandoahOldGC gc(generation, _allow_old_preemption); if (gc.collect(cause)) { // Old collection is complete, the young generation no longer needs this diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp index 4d98d856e6d6d..15fedf1d4f21a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp @@ -164,7 +164,6 @@ void ShenandoahDegenGC::op_degenerated() { { heap->sync_pinned_region_status(); heap->collection_set()->clear_current_index(); - ShenandoahHeapRegion* r; while ((r = heap->collection_set()->next()) != NULL) { if (r->is_pinned()) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index d6695b510302e..64afd5d02c112 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -171,6 +171,14 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah r->set_affiliation(req.affiliation()); r->set_update_watermark(r->bottom()); + + // Any OLD region allocated during concurrent coalesce-and-fill does not need to be coalesced and filled because + // all objects allocated within this region are above TAMS (and thus are implicitly marked). In case this is an + // OLD region and concurrent preparation for mixed evacuations visits this region before the start of the next + // old-gen concurrent mark (i.e. this region is allocated following the start of old-gen concurrent mark but before + // concurrent preparations for mixed evacuations are completed), we mark this region as not requiring any + // coalesce-and-fill processing. This code is only necessary if req.affiliation() is OLD, but harmless if not. + r->finish_coalesce_and_fill(); ctx->capture_top_at_mark_start(r); assert(ctx->top_at_mark_start(r) == r->bottom(), "Newly established allocation region starts with TAMS equal to bottom"); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 087fb645f5909..79d543ac084d3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -501,6 +501,7 @@ ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) : CollectedHeap(), _gc_generation(NULL), _mixed_evac(false), + _prep_for_mixed_evac_in_progress(false), _initial_size(0), _used(0), _committed(0), @@ -1921,6 +1922,16 @@ void ShenandoahHeap::set_concurrent_old_mark_in_progress(bool in_progress) { manage_satb_barrier(in_progress); } +void ShenandoahHeap::set_concurrent_prep_for_mixed_evacuation_in_progress(bool in_progress) { + // Unlike other set-gc-state functions, this may happen outside safepoint. + // Is only set and queried by control thread, so no coherence issues. + _prep_for_mixed_evac_in_progress = in_progress; +} + +bool ShenandoahHeap::is_concurrent_prep_for_mixed_evacuation_in_progress() { + return _prep_for_mixed_evac_in_progress; +} + void ShenandoahHeap::set_aging_cycle(bool in_progress) { _is_aging_cycle.set_cond(in_progress); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index a23450c94bc07..daaeb5bc4be1e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -151,6 +151,7 @@ class ShenandoahHeap : public CollectedHeap { ShenandoahHeapLock _lock; ShenandoahGeneration* _gc_generation; bool _mixed_evac; // true iff most recent evac included at least one old-gen HeapRegion + bool _prep_for_mixed_evac_in_progress; // true iff we are concurrently coalescing and filling old-gen HeapRegions public: ShenandoahHeapLock* lock() { @@ -351,6 +352,7 @@ class ShenandoahHeap : public CollectedHeap { void set_has_forwarded_objects(bool cond); void set_concurrent_strong_root_in_progress(bool cond); void set_concurrent_weak_root_in_progress(bool cond); + void set_concurrent_prep_for_mixed_evacuation_in_progress(bool cond); void set_aging_cycle(bool cond); inline bool is_stable() const; @@ -368,6 +370,7 @@ class ShenandoahHeap : public CollectedHeap { inline bool is_stw_gc_in_progress() const; inline bool is_concurrent_strong_root_in_progress() const; inline bool is_concurrent_weak_root_in_progress() const; + bool is_concurrent_prep_for_mixed_evacuation_in_progress(); inline bool is_aging_cycle() const; private: diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index f0447e02aa653..5ca5892c1e5c4 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -425,12 +425,17 @@ void ShenandoahHeapRegion::print_on(outputStream* st) const { st->cr(); } -// oop_iterate without closure -void ShenandoahHeapRegion::oop_fill_and_coalesce() { - HeapWord* obj_addr = bottom(); +// oop_iterate without closure, return true if completed without cancellation +bool ShenandoahHeapRegion::oop_fill_and_coalesce() { + HeapWord* obj_addr = resume_coalesce_and_fill(); + // Consider yielding to cancel/preemption request after this many coalesce operations (skip marked, or coalesce free). + const size_t preemption_stride = 128; assert(!is_humongous(), "No need to fill or coalesce humongous regions"); - if (!is_active()) return; + if (!is_active()) { + finish_coalesce_and_fill(); + return true; + } ShenandoahHeap* heap = ShenandoahHeap::heap(); ShenandoahMarkingContext* marking_context = heap->marking_context(); @@ -444,6 +449,7 @@ void ShenandoahHeapRegion::oop_fill_and_coalesce() { // Expect marking to be completed before these threads invoke this service. assert(heap->active_generation()->is_mark_complete(), "sanity"); + size_t ops_before_preempt_check = preemption_stride; while (obj_addr < t) { oop obj = cast_to_oop(obj_addr); if (marking_context->is_marked(obj)) { @@ -458,7 +464,17 @@ void ShenandoahHeapRegion::oop_fill_and_coalesce() { heap->card_scan()->coalesce_objects(obj_addr, fill_size); obj_addr = next_marked_obj; } + if (ops_before_preempt_check-- == 0) { + if (heap->cancelled_gc()) { + suspend_coalesce_and_fill(obj_addr); + return false; + } + ops_before_preempt_check = preemption_stride; + } } + // Mark that this region has been coalesced and filled + finish_coalesce_and_fill(); + return true; } void ShenandoahHeapRegion::global_oop_iterate_and_fill_dead(OopIterateClosure* blk) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp index 584e5e69b8852..a7ec4cf4e0f88 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp @@ -240,6 +240,7 @@ class ShenandoahHeapRegion { // Seldom updated fields RegionState _state; + HeapWord* _coalesce_and_fill_boundary; // for old regions not selected as collection set candidates. // Frequently updated fields HeapWord* _top; @@ -368,9 +369,30 @@ class ShenandoahHeapRegion { void recycle(); + inline void reset_coalesce_and_fill_boundary() { + _coalesce_and_fill_boundary = _bottom; + } + + inline void finish_coalesce_and_fill() { + _coalesce_and_fill_boundary = _end; + } + + inline void suspend_coalesce_and_fill(HeapWord* next_focus) { + _coalesce_and_fill_boundary = next_focus; + } + + inline HeapWord* resume_coalesce_and_fill() { + return _coalesce_and_fill_boundary; + } + + inline bool coalesce_and_fill_is_done() { + return _coalesce_and_fill_boundary >= _top; + } + // Coalesce contiguous spans of garbage objects by filling header and reregistering start locations with remembered set. - // This is used by old-gen GC following concurrent marking to make old-gen HeapRegions parseable. - void oop_fill_and_coalesce(); + // This is used by old-gen GC following concurrent marking to make old-gen HeapRegions parseable. Return true iff + // region is completely coalesced and filled. Returns false if cancelled before task is complete. + bool oop_fill_and_coalesce(); // During global collections, this service iterates through an old-gen heap region that is not part of collection // set to fill and register ranges of dead memory. Note that live objects were previously registered. Some dead objects diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.hpp index 56ccaecb52b5f..6b02a6e4d97ce 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.hpp @@ -97,7 +97,6 @@ class ShenandoahHeapRegionCounters : public CHeapObj { private: static jlong encode_heap_status(ShenandoahHeap* heap) ; - }; #endif // SHARE_GC_SHENANDOAH_SHENANDOAHHEAPREGIONCOUNTERS_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp index c238d572d685c..35c96f3b88ffb 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp @@ -40,26 +40,39 @@ class ShenandoahConcurrentCoalesceAndFillTask : public AbstractGangTask { uint _nworkers; ShenandoahHeapRegion** _coalesce_and_fill_region_array; uint _coalesce_and_fill_region_count; + ShenandoahConcurrentGC* _old_gc; + volatile bool _is_preempted; public: - ShenandoahConcurrentCoalesceAndFillTask(uint nworkers, - ShenandoahHeapRegion** coalesce_and_fill_region_array, uint region_count) : + ShenandoahConcurrentCoalesceAndFillTask(uint nworkers, ShenandoahHeapRegion** coalesce_and_fill_region_array, + uint region_count, ShenandoahConcurrentGC* old_gc) : AbstractGangTask("Shenandoah Concurrent Coalesce and Fill"), _nworkers(nworkers), _coalesce_and_fill_region_array(coalesce_and_fill_region_array), - _coalesce_and_fill_region_count(region_count) { + _coalesce_and_fill_region_count(region_count), + _old_gc(old_gc), + _is_preempted(false) { } void work(uint worker_id) { for (uint region_idx = worker_id; region_idx < _coalesce_and_fill_region_count; region_idx += _nworkers) { ShenandoahHeapRegion* r = _coalesce_and_fill_region_array[region_idx]; - if (!r->is_humongous()) - r->oop_fill_and_coalesce(); - else { + if (!r->is_humongous()) { + if (!r->oop_fill_and_coalesce()) { + // Coalesce and fill has been preempted + Atomic::store(&_is_preempted, true); + return; + } + } else { // there's only one object in this region and it's not garbage, so no need to coalesce or fill } } } + + // Value returned from is_completed() is only valid after all worker thread have terminated. + bool is_completed() { + return !Atomic::load(&_is_preempted); + } }; @@ -115,49 +128,55 @@ void ShenandoahOldGC::op_final_mark() { bool ShenandoahOldGC::collect(GCCause::Cause cause) { ShenandoahHeap* heap = ShenandoahHeap::heap(); - // Continue concurrent mark, do not reset regions, do not mark roots, do not collect $200. - _allow_preemption.set(); - entry_mark(); - if (!_allow_preemption.try_unset()) { - // The regulator thread has unset the preemption guard. That thread will shortly cancel - // the gc, but the control thread is now racing it. Wait until this thread sees the cancellation. - while (!heap->cancelled_gc()) { - SpinPause(); + if (!heap->is_concurrent_prep_for_mixed_evacuation_in_progress()) { + // Skip over the initial phases of old collect if we're resuming mixed evacuation preparation. + // Continue concurrent mark, do not reset regions, do not mark roots, do not collect $200. + _allow_preemption.set(); + entry_mark(); + if (!_allow_preemption.try_unset()) { + // The regulator thread has unset the preemption guard. That thread will shortly cancel + // the gc, but the control thread is now racing it. Wait until this thread sees the cancellation. + while (!heap->cancelled_gc()) { + SpinPause(); + } } - } - if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark)) { - return false; - } + if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark)) { + return false; + } - // Complete marking under STW - vmop_entry_final_mark(); + // Complete marking under STW + vmop_entry_final_mark(); - // We aren't dealing with old generation evacuation yet. Our heuristic - // should not have built a cset in final mark. - assert(!heap->is_evacuation_in_progress(), "Old gen evacuations are not supported"); + // We aren't dealing with old generation evacuation yet. Our heuristic + // should not have built a cset in final mark. + assert(!heap->is_evacuation_in_progress(), "Old gen evacuations are not supported"); - // Process weak roots that might still point to regions that would be broken by cleanup - if (heap->is_concurrent_weak_root_in_progress()) { - entry_weak_refs(); - entry_weak_roots(); - } + // Process weak roots that might still point to regions that would be broken by cleanup + if (heap->is_concurrent_weak_root_in_progress()) { + entry_weak_refs(); + entry_weak_roots(); + } - // Final mark might have reclaimed some immediate garbage, kick cleanup to reclaim - // the space. This would be the last action if there is nothing to evacuate. - entry_cleanup_early(); + // Final mark might have reclaimed some immediate garbage, kick cleanup to reclaim + // the space. This would be the last action if there is nothing to evacuate. + entry_cleanup_early(); - { - ShenandoahHeapLocker locker(heap->lock()); - heap->free_set()->log_status(); - } + { + ShenandoahHeapLocker locker(heap->lock()); + heap->free_set()->log_status(); + } - // TODO: Old marking doesn't support class unloading yet - // Perform concurrent class unloading - // if (heap->unload_classes() && - // heap->is_concurrent_weak_root_in_progress()) { - // entry_class_unloading(); - // } + + // TODO: Old marking doesn't support class unloading yet + // Perform concurrent class unloading + // if (heap->unload_classes() && + // heap->is_concurrent_weak_root_in_progress()) { + // entry_class_unloading(); + // } + + heap->set_concurrent_prep_for_mixed_evacuation_in_progress(true); + } // Coalesce and fill objects _after_ weak root processing and class unloading. // Weak root and reference processing makes assertions about unmarked referents @@ -165,7 +184,12 @@ bool ShenandoahOldGC::collect(GCCause::Cause cause) { // a case in the LRB that permits access to from-space objects for the purpose // of class unloading that is unlikely to function correctly if the object has // been filled. - entry_coalesce_and_fill(); + + _allow_preemption.set(); + + if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) { + return false; + } // Prepare for old evacuations (actual evacuations will happen on subsequent young collects). entry_old_evacuations(); @@ -173,6 +197,23 @@ bool ShenandoahOldGC::collect(GCCause::Cause cause) { assert(!heap->is_concurrent_strong_root_in_progress(), "No evacuations during old gc."); vmop_entry_final_roots(); + + if (heap->is_concurrent_prep_for_mixed_evacuation_in_progress()) { + if (!entry_coalesce_and_fill()) { + // If old-gen degenerates instead of resuming, we'll just start up an out-of-cycle degenerated GC. + // This should be a rare event. Normally, we'll resume the coalesce-and-fill effort after the + // preempting young-gen GC finishes. + check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_outside_cycle); + return false; + } + } + if (!_allow_preemption.try_unset()) { + // The regulator thread has unset the preemption guard. That thread will shortly cancel + // the gc, but the control thread is now racing it. Wait until this thread sees the cancellation. + while (!heap->cancelled_gc()) { + SpinPause(); + } + } return true; } @@ -181,30 +222,32 @@ void ShenandoahOldGC::entry_coalesce_and_fill_message(char *buf, size_t len) con jio_snprintf(buf, len, "Coalescing and filling (%s)", _generation->name()); } -void ShenandoahOldGC::op_coalesce_and_fill() { +bool ShenandoahOldGC::op_coalesce_and_fill() { ShenandoahHeap* const heap = ShenandoahHeap::heap(); - + ShenandoahOldHeuristics* old_heuristics = heap->old_heuristics(); WorkGang* workers = heap->workers(); uint nworkers = workers->active_workers(); assert(_generation->generation_mode() == OLD, "Only old-GC does coalesce and fill"); - - ShenandoahOldHeuristics* old_heuristics = heap->old_heuristics(); + log_debug(gc)("Starting (or resuming) coalesce-and-fill of old heap regions"); uint coalesce_and_fill_regions_count = old_heuristics->old_coalesce_and_fill_candidates(); assert(coalesce_and_fill_regions_count <= heap->num_regions(), "Sanity"); old_heuristics->get_coalesce_and_fill_candidates(_coalesce_and_fill_region_array); - ShenandoahConcurrentCoalesceAndFillTask task(nworkers, _coalesce_and_fill_region_array, coalesce_and_fill_regions_count); - - - // TODO: We need to implement preemption of coalesce and fill. If young-gen wants to run while we're working on this, - // we should preempt this code and then resume it after young-gen has finished. This requires that we "remember" the state - // of each worker thread so it can be resumed where it left off. Note that some worker threads may have processed more regions - // than others at the time of preemption. + ShenandoahConcurrentCoalesceAndFillTask task(nworkers, _coalesce_and_fill_region_array, coalesce_and_fill_regions_count, this); workers->run_task(&task); + if (task.is_completed()) { + // Remember that we're done with coalesce-and-fill. + heap->set_concurrent_prep_for_mixed_evacuation_in_progress(false); + return true; + } else { + log_debug(gc)("Suspending coalesce-and-fill of old heap regions"); + // Otherwise, we got preempted before the work was done. + return false; + } } -void ShenandoahOldGC::entry_coalesce_and_fill() { +bool ShenandoahOldGC::entry_coalesce_and_fill() { char msg[1024]; ShenandoahHeap* const heap = ShenandoahHeap::heap(); @@ -217,5 +260,5 @@ void ShenandoahOldGC::entry_coalesce_and_fill() { ShenandoahWorkerPolicy::calc_workers_for_conc_marking(), "concurrent coalesce and fill"); - op_coalesce_and_fill(); + return op_coalesce_and_fill(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.hpp index f4de5673d665c..d62f7d88371c3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.hpp @@ -43,9 +43,9 @@ class ShenandoahOldGC : public ShenandoahConcurrentGC { ShenandoahHeapRegion** _coalesce_and_fill_region_array; void entry_old_evacuations(); - void entry_coalesce_and_fill(); + bool entry_coalesce_and_fill(); ShenandoahSharedFlag& _allow_preemption; - void op_coalesce_and_fill(); + bool op_coalesce_and_fill(); void entry_coalesce_and_fill_message(char *buf, size_t len) const; }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp index 4b04effef8803..1a9c2cdc83bd8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp @@ -43,9 +43,11 @@ const char* ShenandoahYoungGeneration::name() const { void ShenandoahYoungGeneration::set_concurrent_mark_in_progress(bool in_progress) { ShenandoahHeap* heap = ShenandoahHeap::heap(); heap->set_concurrent_young_mark_in_progress(in_progress); - if (_old_gen_task_queues != nullptr && in_progress) { + if (_old_gen_task_queues != nullptr && in_progress && !heap->is_concurrent_prep_for_mixed_evacuation_in_progress()) { // This is not a bug. When the young generation marking is complete, - // the old generation marking is still in progress. + // the old generation marking is still in progress, unless it's not. + // In the case that old-gen preparation for mixed evacuation has been + // preempted, we do not want to set concurrent old mark to be in progress. heap->set_concurrent_old_mark_in_progress(in_progress); } } From 6d8ebd70eaa24127b33787a9ba48b95e56246061 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Thu, 14 Oct 2021 22:29:23 +0000 Subject: [PATCH 083/254] Do not coalesce dead objects in global cycle if GC was cancelled Reviewed-by: kdnilsen --- src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index b894daf38d02a..c0395c39234a3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -156,7 +156,7 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) { entry_strong_roots(); } - if (heap->mode()->is_generational() && _generation->generation_mode() == GLOBAL) { + if (!heap->cancelled_gc() && heap->mode()->is_generational() && _generation->generation_mode() == GLOBAL) { entry_global_coalesce_and_fill(); } From 8caba42835369dc25105850f7022497c7ec0bf41 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Thu, 14 Oct 2021 22:58:38 +0000 Subject: [PATCH 084/254] Replace 'HEY!' comments with TODO and remove unnecessary usages Reviewed-by: kdnilsen --- .../share/gc/shenandoah/shenandoahBarrierSet.inline.hpp | 2 +- .../share/gc/shenandoah/shenandoahControlThread.cpp | 7 +++---- src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp | 2 -- src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp | 2 +- src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.cpp | 7 +++---- .../share/gc/shenandoah/shenandoahScanRemembered.hpp | 4 ++-- 6 files changed, 10 insertions(+), 14 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp index 612955c4e2d96..1d5b231d6a6e0 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp @@ -417,7 +417,7 @@ void ShenandoahBarrierSet::arraycopy_barrier(T* src, T* dst, size_t count) { if (_heap->mode()->is_generational()) { assert(ShenandoahSATBBarrier, "Generational mode assumes SATB mode"); - // HEY! Could we optimize here by checking that dst is in an old region? + // TODO: Could we optimize here by checking that dst is in an old region? if ((gc_state & ShenandoahHeap::OLD_MARKING) != 0) { // Note that we can't do the arraycopy marking using the 'src' array when // SATB mode is enabled (so we can't do this as part of the iteration for diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index 22ed22c03198d..9fc52130f6a02 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -102,11 +102,10 @@ void ShenandoahControlThread::run_service() { ShenandoahCollectorPolicy* policy = heap->shenandoah_policy(); - // HEY! heuristics are notified of allocation failures here and other outcomes + // Heuristics are notified of allocation failures here and other outcomes // of the cycle. They're also used here to control whether the Nth consecutive - // degenerated cycle should be 'promoted' to a full cycle. This changes the - // threading model for them somewhat, as they are now evaluated on a separate - // thread. + // degenerated cycle should be 'promoted' to a full cycle. The decision to + // trigger a cycle or not is evaluated on the regulator thread. ShenandoahHeuristics* global_heuristics = heap->global_generation()->heuristics(); while (!in_graceful_shutdown() && !should_terminate()) { // Figure out if we have pending requests. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp index 036f45d87f1b4..786321cd36776 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp @@ -573,8 +573,6 @@ inline void ShenandoahHeap::marked_object_iterate(ShenandoahHeapRegion* region, assert(! region->is_humongous_continuation(), "no humongous continuation regions here"); ShenandoahMarkingContext* const ctx = marking_context(); - // HEY! All callers (at the time of this writing) have already asserted the mark context is complete. - // assert(ctx->is_complete(), "sanity"); HeapWord* tams = ctx->top_at_mark_start(region); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp index bb05ccb44beaa..34940f83f0e9a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp @@ -58,7 +58,7 @@ template void ShenandoahMark::do_task(ShenandoahObjToScanQueue* q, T* cl, ShenandoahLiveData* live_data, StringDedup::Requests* const req, ShenandoahMarkTask* task) { oop obj = task->obj(); - // HEY! This will push array chunks into the mark queue with no regard for + // TODO: This will push array chunks into the mark queue with no regard for // generations. I don't think it will break anything, but the young generation // scan might end up processing some old generation array chunks. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.cpp index 8d2c5492f0c59..4d3f12b94f035 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.cpp @@ -61,12 +61,11 @@ void ShenandoahFinalMarkUpdateRegionStateClosure::heap_region_do(ShenandoahHeapR } if (ShenandoahHeap::heap()->mode()->is_generational()) { - // HEY! Allocations move the watermark when top moves, however compacting + // Allocations move the watermark when top moves, however compacting // objects will sometimes lower top beneath the watermark, after which, // attempts to read the watermark will assert out (watermark should not be - // higher than top). I think the right way™ to check for new allocations - // is to compare top with the TAMS as is done earlier in this function. - // if (r->top() != r->get_update_watermark()) { + // higher than top). The right way™ to check for new allocations is to compare + // top with the TAMS as is done earlier in this function. if (top > tams) { // There have been allocations in this region since the start of the cycle. // Any objects new to this region must not assimilate elevated age. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp index bd946a8408cc7..70f9685042ed1 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp @@ -538,7 +538,7 @@ class ShenandoahCardCluster: public CHeapObj { ShenandoahCardCluster(RememberedSet *rs) { _rs = rs; - // HEY! We don't really need object_starts entries for every card entry. We only need these for + // TODO: We don't really need object_starts entries for every card entry. We only need these for // the card entries that correspond to old-gen memory. But for now, let's be quick and dirty. object_starts = (uint16_t *) malloc(rs->total_cards() * sizeof(uint16_t)); if (object_starts == NULL) @@ -875,7 +875,7 @@ class ShenandoahScanRemembered: public CHeapObj { delete _scc; } - // HEY! We really don't want to share all of these APIs with arbitrary consumers of the ShenandoahScanRemembered abstraction. + // TODO: We really don't want to share all of these APIs with arbitrary consumers of the ShenandoahScanRemembered abstraction. // But in the spirit of quick and dirty for the time being, I'm going to go ahead and publish everything for right now. Some // of existing code already depends on having access to these services (because existing code has not been written to honor // full abstraction of remembered set scanning. In the not too distant future, we want to try to make most, if not all, of From 8c87aa9c0122490cbb3760b524d5024fa8a0df3b Mon Sep 17 00:00:00 2001 From: William Kemper Date: Fri, 15 Oct 2021 16:10:07 +0000 Subject: [PATCH 085/254] Reset coalesce and fill boundary during global collect Reviewed-by: kdnilsen --- src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 79d543ac084d3..5d12bbdd327fa 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -966,6 +966,11 @@ void ShenandoahHeap::coalesce_and_fill_old_regions() { virtual void heap_region_do(ShenandoahHeapRegion* region) override { // old region is not in the collection set and was not immediately trashed if (region->is_old() && region->is_active() && !region->is_humongous()) { + // Reset the coalesce and fill boundary because this is a global collect + // and cannot be preempted by young collects. We want to be sure the entire + // region is coalesced here and does not resume from a previously interrupted + // or completed coalescing. + region->reset_coalesce_and_fill_boundary(); region->oop_fill_and_coalesce(); } } From ca939b69f95db2cb8ead9c9459b2d7712909a4b6 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Tue, 19 Oct 2021 21:33:21 +0000 Subject: [PATCH 086/254] Degenerate to young cycles for out of cycle allocation failures Reviewed-by: kdnilsen --- .../gc/shenandoah/shenandoahConcurrentGC.cpp | 3 +- .../gc/shenandoah/shenandoahControlThread.cpp | 2 +- .../gc/shenandoah/shenandoahDegeneratedGC.cpp | 37 ++++++++----------- .../gc/shenandoah/shenandoahDegeneratedGC.hpp | 2 +- .../share/gc/shenandoah/shenandoahGC.cpp | 2 + .../share/gc/shenandoah/shenandoahGC.hpp | 1 + .../gc/shenandoah/shenandoahGeneration.cpp | 2 +- .../share/gc/shenandoah/shenandoahHeap.cpp | 23 +++++++++++- .../gc/shenandoah/shenandoahHeap.inline.hpp | 4 +- .../share/gc/shenandoah/shenandoahSTWMark.cpp | 8 +++- 10 files changed, 52 insertions(+), 32 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index c0395c39234a3..d0b175584b5b4 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -107,12 +107,13 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) { // Concurrent remembered set scanning if (_generation->generation_mode() == YOUNG) { + ShenandoahConcurrentPhase gc_phase("Concurrent remembered set scanning", ShenandoahPhaseTimings::init_scan_rset); _generation->scan_remembered_set(); } // Concurrent mark roots entry_mark_roots(); - if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_outside_cycle)) return false; + if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_roots)) return false; // Continue concurrent mark entry_mark(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index 9fc52130f6a02..4567332fb566d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -135,7 +135,7 @@ void ShenandoahControlThread::run_service() { _degen_point = ShenandoahGC::_degenerated_outside_cycle; if (degen_point == ShenandoahGC::_degenerated_outside_cycle) { - _degen_generation = heap->global_generation(); + _degen_generation = heap->mode()->is_generational() ? heap->young_generation() : heap->global_generation(); } else { assert(_degen_generation != NULL, "Need to know which generation to resume."); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp index 15fedf1d4f21a..e8f6a816c3730 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp @@ -63,7 +63,8 @@ void ShenandoahDegenGC::vmop_degenerated() { } void ShenandoahDegenGC::entry_degenerated() { - const char* msg = degen_event_message(_degen_point); + char msg[1024]; + degen_event_message(_degen_point, msg, sizeof(msg)); ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::degen_gc, true /* log_heap_usage */); EventMark em("%s", msg); ShenandoahHeap* const heap = ShenandoahHeap::heap(); @@ -110,17 +111,23 @@ void ShenandoahDegenGC::op_degenerated() { // space. It makes little sense to wait for Full GC to reclaim as much as it can, when // we can do the most aggressive degen cycle, which includes processing references and // class unloading, unless those features are explicitly disabled. - // + + // Note that we can only do this for "outside-cycle" degens, otherwise we would risk + // changing the cycle parameters mid-cycle during concurrent -> degenerated handover. + heap->set_unload_classes((!heap->mode()->is_generational() || _generation->generation_mode() == GLOBAL) && _generation->heuristics()->can_unload_classes()); + + if (_generation->generation_mode() == YOUNG || (_generation->generation_mode() == GLOBAL && ShenandoahVerify)) { + // Swap remembered sets for young, or if the verifier will run during a global collect + _generation->swap_remembered_set(); + } + + case _degenerated_roots: // Degenerated from concurrent root mark, reset the flag for STW mark if (heap->is_concurrent_mark_in_progress()) { heap->cancel_concurrent_mark(); } - // Note that we can only do this for "outside-cycle" degens, otherwise we would risk - // changing the cycle parameters mid-cycle during concurrent -> degenerated handover. - heap->set_unload_classes(_generation->heuristics()->can_unload_classes()); - op_reset(); // STW mark @@ -354,22 +361,8 @@ void ShenandoahDegenGC::op_degenerated_futile() { full_gc.op_full(GCCause::_shenandoah_upgrade_to_full_gc); } -const char* ShenandoahDegenGC::degen_event_message(ShenandoahDegenPoint point) const { - switch (point) { - case _degenerated_unset: - return "Pause Degenerated GC ()"; - case _degenerated_outside_cycle: - return "Pause Degenerated GC (Outside of Cycle)"; - case _degenerated_mark: - return "Pause Degenerated GC (Mark)"; - case _degenerated_evac: - return "Pause Degenerated GC (Evacuation)"; - case _degenerated_updaterefs: - return "Pause Degenerated GC (Update Refs)"; - default: - ShouldNotReachHere(); - return "ERROR"; - } +void ShenandoahDegenGC::degen_event_message(ShenandoahDegenPoint point, char* buf, size_t len) const { + jio_snprintf(buf, len, "Pause Degenerated %s GC (%s)", _generation->name(), ShenandoahGC::degen_point_to_string(point)); } void ShenandoahDegenGC::upgrade_to_full() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.hpp index 53bf76a4f8c3d..7b0ffe253ac30 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.hpp @@ -63,7 +63,7 @@ class ShenandoahDegenGC : public ShenandoahGC { void op_degenerated_futile(); void op_degenerated_fail(); - const char* degen_event_message(ShenandoahDegenPoint point) const; + void degen_event_message(ShenandoahDegenPoint point, char* buf, size_t len) const; void upgrade_to_full(); }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGC.cpp index c0ab9a61f813c..25a0df0eb891a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGC.cpp @@ -39,6 +39,8 @@ const char* ShenandoahGC::degen_point_to_string(ShenandoahDegenPoint point) { return ""; case _degenerated_outside_cycle: return "Outside of Cycle"; + case _degenerated_roots: + return "Roots"; case _degenerated_mark: return "Mark"; case _degenerated_evac: diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGC.hpp index e0d3724723a29..4e929363c947a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGC.hpp @@ -50,6 +50,7 @@ class ShenandoahGC : public StackObj { enum ShenandoahDegenPoint { _degenerated_unset, _degenerated_outside_cycle, + _degenerated_roots, _degenerated_mark, _degenerated_evac, _degenerated_updaterefs, diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index f4c09357b91b1..27473528f4dba 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -161,6 +161,7 @@ void ShenandoahGeneration::reset_mark_bitmap() { // location of the card table. So the interim implementation of swap_remembered_set will copy the write-table // onto the read-table and will then clear the write-table. void ShenandoahGeneration::swap_remembered_set() { + // Must be sure that marking is complete before we swap remembered set. ShenandoahHeap* heap = ShenandoahHeap::heap(); heap->assert_gc_workers(heap->workers()->active_workers()); shenandoah_assert_safepoint(); @@ -309,7 +310,6 @@ void ShenandoahGeneration::scan_remembered_set() { uint nworkers = heap->workers()->active_workers(); reserve_task_queues(nworkers); - ShenandoahConcurrentPhase gc_phase("Concurrent remembered set scanning", ShenandoahPhaseTimings::init_scan_rset); ShenandoahReferenceProcessor* rp = ref_processor(); ShenandoahRegionIterator regions; ShenandoahScanRememberedTask task(task_queues(), old_gen_task_queues(), rp, ®ions); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 5d12bbdd327fa..6a27fe25f991b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -1103,7 +1103,28 @@ HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) { HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req, bool& in_new_region) { ShenandoahHeapLocker locker(lock()); - return _free_set->allocate(req, in_new_region); + HeapWord* result = _free_set->allocate(req, in_new_region); + if (result != NULL && req.affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) { + // Register the newly allocated object while we're holding the global lock since there's no synchronization + // built in to the implementation of register_object(). There are potential races when multiple independent + // threads are allocating objects, some of which might span the same card region. For example, consider + // a card table's memory region within which three objects are being allocated by three different threads: + // + // objects being "concurrently" allocated: + // [-----a------][-----b-----][--------------c------------------] + // [---- card table memory range --------------] + // + // Before any objects are allocated, this card's memory range holds no objects. Note that: + // allocation of object a wants to set the has-object, first-start, and last-start attributes of the preceding card region. + // allocation of object b wants to set the has-object, first-start, and last-start attributes of this card region. + // allocation of object c also wants to set the has-object, first-start, and last-start attributes of this card region. + // + // The thread allocating b and the thread allocating c can "race" in various ways, resulting in confusion, such as last-start + // representing object b while first-start represents object c. This is why we need to require all register_object() + // invocations to be "mutually exclusive" with respect to each card's memory range. + ShenandoahHeap::heap()->card_scan()->register_object(result); + } + return result; } HeapWord* ShenandoahHeap::mem_allocate(size_t size, diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp index 786321cd36776..a9364bf4ff687 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp @@ -461,9 +461,7 @@ inline oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, Shenandoah // we have to keep the fwdptr initialized and pointing to our (stale) copy. fill_with_object(copy, size); shenandoah_assert_correct(NULL, copy_val); - if (mode()->is_generational() && target_gen == OLD_GENERATION) { - card_scan()->register_object(copy); - } + // For non-LAB allocations, the object has already been registered } shenandoah_assert_correct(NULL, result); return result; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp index f01e07d5769fc..7955e338c44d1 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp @@ -114,6 +114,11 @@ void ShenandoahSTWMark::mark() { { // Mark + if (_generation->generation_mode() == YOUNG) { + // But only scan the remembered set for young generation. + _generation->scan_remembered_set(); + } + StrongRootsScope scope(nworkers); ShenandoahSTWMarkTask task(this); heap->workers()->run_task(&task); @@ -121,7 +126,7 @@ void ShenandoahSTWMark::mark() { assert(task_queues()->is_empty(), "Should be empty"); } - heap->global_generation()->set_mark_complete(); + _generation->set_mark_complete(); assert(task_queues()->is_empty(), "Should be empty"); TASKQUEUE_STATS_ONLY(task_queues()->print_taskqueue_stats()); @@ -138,7 +143,6 @@ void ShenandoahSTWMark::mark_roots(uint worker_id) { case YOUNG: { ShenandoahInitMarkRootsClosure init_mark(task_queues()->queue(worker_id)); _root_scanner.roots_do(&init_mark, worker_id); - _generation->scan_remembered_set(); break; } default: From e2c8047cfc83fd4932d64f2ea393fd8bf27ea0d7 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Tue, 19 Oct 2021 23:47:11 +0000 Subject: [PATCH 087/254] Initialize promotion failed member to false Reviewed-by: kdnilsen --- .../share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp index c8959fb2e0738..5199f88e659ab 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp @@ -35,7 +35,8 @@ ShenandoahOldHeuristics::ShenandoahOldHeuristics(ShenandoahGeneration* generatio _hidden_next_old_collection_candidate(0), _old_coalesce_and_fill_candidates(0), _first_coalesce_and_fill_candidate(0), - _trigger_heuristic(trigger_heuristic) + _trigger_heuristic(trigger_heuristic), + _promotion_failed(false) { } From 7bc0d2bfd7cbff829e40c5bcbdfa71980ce1f3ba Mon Sep 17 00:00:00 2001 From: William Kemper Date: Thu, 21 Oct 2021 20:28:40 +0000 Subject: [PATCH 088/254] Make region sampling aware of full collections Reviewed-by: kdnilsen --- .../share/gc/shenandoah/shenandoahHeapRegionCounters.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp index f31e5495a1efa..687c3ffc8e1ad 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp @@ -124,13 +124,13 @@ void ShenandoahHeapRegionCounters::update() { } static int encode_phase(ShenandoahHeap* heap) { - if (heap->is_evacuation_in_progress()) { + if (heap->is_evacuation_in_progress() || heap->is_full_gc_move_in_progress()) { return 2; } - if (heap->is_update_refs_in_progress()) { + if (heap->is_update_refs_in_progress() || heap->is_full_gc_move_in_progress()) { return 3; } - if (heap->is_concurrent_mark_in_progress()) { + if (heap->is_concurrent_mark_in_progress() || heap->is_full_gc_in_progress()) { return 1; } assert(heap->is_idle(), "What is it doing?"); @@ -150,7 +150,7 @@ static int get_generation_shift(ShenandoahGeneration* generation) { jlong ShenandoahHeapRegionCounters::encode_heap_status(ShenandoahHeap* heap) { - if (heap->is_idle()) { + if (heap->is_idle() && !heap->is_full_gc_in_progress()) { return 0; } From e442388f07101581cad97fe2052530f3d47ad794 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Thu, 21 Oct 2021 20:29:43 +0000 Subject: [PATCH 089/254] Enforce maximum capacity of young generation when allocating Reviewed-by: kdnilsen --- src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp | 4 ++++ .../share/gc/shenandoah/shenandoahInitLogger.cpp | 14 ++++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 6a27fe25f991b..8f3cff6ceebcf 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -1102,6 +1102,10 @@ HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) { } HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req, bool& in_new_region) { + if (mode()->is_generational() && req.affiliation() == YOUNG_GENERATION && young_generation()->used() + req.size() >= young_generation()->max_capacity()) { + return nullptr; + } + ShenandoahHeapLocker locker(lock()); HeapWord* result = _free_set->allocate(req, in_new_region); if (result != NULL && req.affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahInitLogger.cpp b/src/hotspot/share/gc/shenandoah/shenandoahInitLogger.cpp index 27ea64f587778..472883e33e167 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahInitLogger.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahInitLogger.cpp @@ -27,6 +27,7 @@ #include "gc/shenandoah/shenandoahHeapRegion.hpp" #include "gc/shenandoah/shenandoahInitLogger.hpp" #include "gc/shenandoah/shenandoahGeneration.hpp" +#include "gc/shenandoah/shenandoahYoungGeneration.hpp" #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" #include "gc/shenandoah/mode/shenandoahMode.hpp" #include "logging/log.hpp" @@ -41,8 +42,17 @@ void ShenandoahInitLogger::print_heap() { log_info(gc, init)("Mode: %s", heap->mode()->name()); - log_info(gc, init)("Heuristics: %s", - heap->global_generation()->heuristics()->name()); + if (!heap->mode()->is_generational()) { + log_info(gc, init)("Heuristics: %s", heap->global_generation()->heuristics()->name()); + } else { + log_info(gc, init)("Young Heuristics: %s", heap->young_generation()->heuristics()->name()); + log_info(gc, init)("Old Heuristics: %s", heap->old_generation()->heuristics()->name()); + log_info(gc, init)("Young Generation Max: " SIZE_FORMAT "%s", + byte_size_in_proper_unit(heap->young_generation()->max_capacity()), + proper_unit_for_byte_size(heap->young_generation()->max_capacity())); + } + + log_info(gc, init)("Heap Region Count: " SIZE_FORMAT, ShenandoahHeapRegion::region_count()); From ad4ad1fb571b0c34c3bd4de492b0e0f9214612c8 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Wed, 27 Oct 2021 20:17:58 +0000 Subject: [PATCH 090/254] Align plabs on card boundaries Reviewed-by: wkemper --- .../share/gc/shenandoah/shenandoahFreeSet.cpp | 57 ++++++++++++++++--- .../share/gc/shenandoah/shenandoahHeap.cpp | 8 ++- .../gc/shenandoah/shenandoahHeapRegion.hpp | 3 + .../shenandoahHeapRegion.inline.hpp | 34 +++++++++++ 4 files changed, 94 insertions(+), 8 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index 64afd5d02c112..018624c8df078 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -198,21 +198,64 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah HeapWord* result = NULL; size_t size = req.size(); + // req.size() is in words, free() is in bytes. if (ShenandoahElasticTLAB && req.is_lab_alloc()) { - size_t free = align_down(r->free() >> LogHeapWordSize, MinObjAlignment); - if (size > free) { - size = free; + if (req.type() == ShenandoahAllocRequest::_alloc_plab) { + // Need to assure that plabs are aligned on multiple of card region. + size_t free = r->free(); + size_t usable_free = (free / CardTable::card_size) << CardTable::card_shift; + free /= HeapWordSize; + usable_free /= HeapWordSize; + if (size > usable_free) { + size = usable_free; + } + if (size >= req.min_size()) { + result = r->allocate_aligned(size, req, CardTable::card_size); + assert (result != NULL, "Allocation must succeed: free " SIZE_FORMAT ", actual " SIZE_FORMAT, usable_free, size); + if (free > usable_free) { + // Account for the alignment padding + size_t padding = (free - usable_free) * HeapWordSize; + increase_used(padding); + assert(r->affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION, "All PLABs reside in old-gen"); + _heap->old_generation()->increase_used(padding); + // For verification consistency, we need to report this padding to _heap + _heap->increase_used(padding); + } + } + } else { + size_t free = align_down(r->free() >> LogHeapWordSize, MinObjAlignment); + if (size > free) { + size = free; + } + if (size >= req.min_size()) { + result = r->allocate(size, req); + assert (result != NULL, "Allocation must succeed: free " SIZE_FORMAT ", actual " SIZE_FORMAT, free, size); + } } - if (size >= req.min_size()) { - result = r->allocate(size, req); - assert (result != NULL, "Allocation must succeed: free " SIZE_FORMAT ", actual " SIZE_FORMAT, free, size); + } else if (req.is_lab_alloc() && req.type() == ShenandoahAllocRequest::_alloc_plab) { + size_t free = r->free(); + size_t usable_free = (free / CardTable::card_size) << CardTable::card_shift; + free /= HeapWordSize; + usable_free /= HeapWordSize; + if (size <= usable_free) { + assert(size % CardTable::card_size_in_words == 0, "PLAB size must be multiple of remembered set card size"); + + result = r->allocate_aligned(size, req, CardTable::card_size); + assert (result != NULL, "Allocation must succeed: free " SIZE_FORMAT ", actual " SIZE_FORMAT, usable_free, size); + + // Account for the alignment padding + size_t padding = (free - usable_free) * HeapWordSize; + increase_used(padding); + assert(r->affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION, "All PLABs reside in old-gen"); + _heap->old_generation()->increase_used(padding); + // For verification consistency, we need to report this padding to _heap + _heap->increase_used(padding); } } else { result = r->allocate(size, req); } if (result != NULL) { - // Record actual allocation size req.set_actual_size(size); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 8f3cff6ceebcf..004d9e315297b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -894,9 +894,15 @@ HeapWord* ShenandoahHeap::allocate_from_plab_slow(Thread* thread, size_t size) { new_size = MIN2(new_size, PLAB::max_size()); new_size = MAX2(new_size, PLAB::min_size()); + size_t unalignment = new_size % CardTable::card_size_in_words; + if (unalignment != 0) { + new_size = new_size - unalignment + CardTable::card_size_in_words; + } + // Record new heuristic value even if we take any shortcut. This captures // the case when moderately-sized objects always take a shortcut. At some point, - // heuristics should catch up with them. + // heuristics should catch up with them. Note that the requested new_size may + // not be honored, but we remember that this is the preferred size. ShenandoahThreadLocalData::set_plab_size(thread, new_size); if (new_size < size) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp index a7ec4cf4e0f88..ace6bcf3e3d61 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp @@ -347,6 +347,9 @@ class ShenandoahHeapRegion { return _index; } + // Allocation (return NULL if full) + inline HeapWord* allocate_aligned(size_t word_size, ShenandoahAllocRequest req, size_t alignment_in_words); + // Allocation (return NULL if full) inline HeapWord* allocate(size_t word_size, ShenandoahAllocRequest req); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp index 94de80b05edbc..fde058fca8ad2 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp @@ -31,6 +31,40 @@ #include "gc/shenandoah/shenandoahPacer.inline.hpp" #include "runtime/atomic.hpp" +// If next available memory is not aligned on address that is multiple of alignment, fill the empty space +// so that returned object is aligned on an address that is a multiple of alignment_in_words. Requested +// size is in words. +HeapWord* ShenandoahHeapRegion::allocate_aligned(size_t size, ShenandoahAllocRequest req, size_t alignment_in_bytes) { + shenandoah_assert_heaplocked_or_safepoint(); + assert(is_object_aligned(size), "alloc size breaks alignment: " SIZE_FORMAT, size); + + HeapWord* obj = top(); + uintptr_t addr_as_int = (uintptr_t) obj; + + size_t unalignment_bytes = addr_as_int % alignment_in_bytes; + size_t unalignment_words = unalignment_bytes / HeapWordSize; + if (pointer_delta(end(), obj + unalignment_words) >= size) { + if (unalignment_words > 0) { + size_t pad_words = (alignment_in_bytes / HeapWordSize) - unalignment_words; + ShenandoahHeap::fill_with_object(obj, pad_words); + ShenandoahHeap::heap()->card_scan()->register_object(obj); + obj += pad_words; + } + + make_regular_allocation(req.affiliation()); + adjust_alloc_metadata(req.type(), size); + + HeapWord* new_top = obj + size; + set_top(new_top); + assert(is_object_aligned(new_top), "new top breaks alignment: " PTR_FORMAT, p2i(new_top)); + assert(((uintptr_t) obj) % (alignment_in_bytes) == 0, "obj is not aligned: " PTR_FORMAT, p2i(obj)); + + return obj; + } else { + return NULL; + } +} + HeapWord* ShenandoahHeapRegion::allocate(size_t size, ShenandoahAllocRequest req) { shenandoah_assert_heaplocked_or_safepoint(); assert(is_object_aligned(size), "alloc size breaks alignment: " SIZE_FORMAT, size); From 5cba4ba9546243734ec35946cb93e0498cc6f0b4 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Mon, 1 Nov 2021 16:17:14 +0000 Subject: [PATCH 091/254] Fix preempt old preparation Reviewed-by: wkemper --- .../shenandoah/heuristics/shenandoahOldHeuristics.cpp | 10 +++++++--- .../shenandoah/heuristics/shenandoahOldHeuristics.hpp | 3 +++ src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp | 2 +- src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp | 8 +++++--- .../share/gc/shenandoah/shenandoahHeapRegion.cpp | 4 ++-- .../share/gc/shenandoah/shenandoahHeapRegion.hpp | 8 ++------ src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp | 9 +++++---- src/hotspot/share/gc/shenandoah/shenandoahOldGC.hpp | 2 +- .../gc/shenandoah/shenandoahScanRemembered.inline.hpp | 2 +- 9 files changed, 27 insertions(+), 21 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp index 5199f88e659ab..8d8f9e13faa14 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp @@ -182,12 +182,12 @@ void ShenandoahOldHeuristics::prepare_for_old_collections() { total_garbage += garbage; if (region->is_regular()) { - region->reset_coalesce_and_fill_boundary(); if (!region->has_live()) { region->make_trash_immediate(); immediate_regions++; immediate_garbage += garbage; } else { + region->begin_preemptible_coalesce_and_fill(); candidates[cand_idx]._region = region; candidates[cand_idx]._garbage = garbage; cand_idx++; @@ -265,10 +265,14 @@ void ShenandoahOldHeuristics::start_old_evacuations() { _hidden_old_collection_candidates = 0; } +uint ShenandoahOldHeuristics::unprocessed_old_or_hidden_collection_candidates() { + assert(_generation->generation_mode() == OLD, "This service only available for old-gc heuristics"); + return _old_collection_candidates + _hidden_old_collection_candidates; +} uint ShenandoahOldHeuristics::unprocessed_old_collection_candidates() { assert(_generation->generation_mode() == OLD, "This service only available for old-gc heuristics"); - return _old_collection_candidates + _hidden_old_collection_candidates; + return _old_collection_candidates; } ShenandoahHeapRegion* ShenandoahOldHeuristics::next_old_collection_candidate() { @@ -324,7 +328,7 @@ bool ShenandoahOldHeuristics::should_start_gc() { // Future refinement: under certain circumstances, we might be more sophisticated about this choice. // For example, we could choose to abandon the previous old collection before it has completed evacuations, // but this would require that we coalesce and fill all garbage within unevacuated collection-set regions. - if (unprocessed_old_collection_candidates() > 0) { + if (unprocessed_old_or_hidden_collection_candidates() > 0) { return false; } diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp index a20f6df0790ca..2dee836d8a2bd 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp @@ -92,6 +92,9 @@ class ShenandoahOldHeuristics : public ShenandoahHeuristics { // How many old-collection candidates have not yet been processed? uint unprocessed_old_collection_candidates(); + // How many old or hidden collection candidates have not yet been processed? + uint unprocessed_old_or_hidden_collection_candidates(); + // Return the next old-collection candidate in order of decreasing amounts of garbage. (We process most-garbage regions // first.) This does not consume the candidate. If the candidate is selected for inclusion in a collection set, then // the candidate is consumed by invoking consume_old_collection_candidate(). diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index 018624c8df078..308314cb9b640 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -178,7 +178,7 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah // old-gen concurrent mark (i.e. this region is allocated following the start of old-gen concurrent mark but before // concurrent preparations for mixed evacuations are completed), we mark this region as not requiring any // coalesce-and-fill processing. This code is only necessary if req.affiliation() is OLD, but harmless if not. - r->finish_coalesce_and_fill(); + r->end_preemptible_coalesce_and_fill(); ctx->capture_top_at_mark_start(r); assert(ctx->top_at_mark_start(r) == r->bottom(), "Newly established allocation region starts with TAMS equal to bottom"); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 004d9e315297b..53d2370963f10 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -976,7 +976,7 @@ void ShenandoahHeap::coalesce_and_fill_old_regions() { // and cannot be preempted by young collects. We want to be sure the entire // region is coalesced here and does not resume from a previously interrupted // or completed coalescing. - region->reset_coalesce_and_fill_boundary(); + region->begin_preemptible_coalesce_and_fill(); region->oop_fill_and_coalesce(); } } @@ -2712,7 +2712,8 @@ void ShenandoahHeap::verify_rem_set_at_mark() { log_debug(gc)("Verifying remembered set at %s mark", doing_mixed_evacuations()? "mixed": "young"); - if (doing_mixed_evacuations() || active_generation()->generation_mode() == GLOBAL) { + if (doing_mixed_evacuations() || + is_concurrent_prep_for_mixed_evacuation_in_progress() || active_generation()->generation_mode() == GLOBAL) { ctx = complete_marking_context(); } else { ctx = nullptr; @@ -2846,7 +2847,8 @@ void ShenandoahHeap::verify_rem_set_at_update_ref() { ShenandoahRegionIterator iterator; ShenandoahMarkingContext* ctx; - if (doing_mixed_evacuations() || active_generation()->generation_mode() == GLOBAL) { + if (doing_mixed_evacuations() || + is_concurrent_prep_for_mixed_evacuation_in_progress() || active_generation()->generation_mode() == GLOBAL) { ctx = complete_marking_context(); } else { ctx = nullptr; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index 5ca5892c1e5c4..505edf23c2492 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -433,7 +433,7 @@ bool ShenandoahHeapRegion::oop_fill_and_coalesce() { assert(!is_humongous(), "No need to fill or coalesce humongous regions"); if (!is_active()) { - finish_coalesce_and_fill(); + end_preemptible_coalesce_and_fill(); return true; } @@ -473,7 +473,7 @@ bool ShenandoahHeapRegion::oop_fill_and_coalesce() { } } // Mark that this region has been coalesced and filled - finish_coalesce_and_fill(); + end_preemptible_coalesce_and_fill(); return true; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp index ace6bcf3e3d61..208af1e894933 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp @@ -372,11 +372,11 @@ class ShenandoahHeapRegion { void recycle(); - inline void reset_coalesce_and_fill_boundary() { + inline void begin_preemptible_coalesce_and_fill() { _coalesce_and_fill_boundary = _bottom; } - inline void finish_coalesce_and_fill() { + inline void end_preemptible_coalesce_and_fill() { _coalesce_and_fill_boundary = _end; } @@ -388,10 +388,6 @@ class ShenandoahHeapRegion { return _coalesce_and_fill_boundary; } - inline bool coalesce_and_fill_is_done() { - return _coalesce_and_fill_boundary >= _top; - } - // Coalesce contiguous spans of garbage objects by filling header and reregistering start locations with remembered set. // This is used by old-gen GC following concurrent marking to make old-gen HeapRegions parseable. Return true iff // region is completely coalesced and filled. Returns false if cancelled before task is complete. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp index 35c96f3b88ffb..99420cfa7ec05 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp @@ -81,7 +81,7 @@ ShenandoahOldGC::ShenandoahOldGC(ShenandoahGeneration* generation, ShenandoahSha _coalesce_and_fill_region_array = NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, ShenandoahHeap::heap()->num_regions(), mtGC); } -void ShenandoahOldGC::entry_old_evacuations() { +void ShenandoahOldGC::start_old_evacuations() { ShenandoahHeap* heap = ShenandoahHeap::heap(); ShenandoahOldHeuristics* old_heuristics = heap->old_heuristics(); old_heuristics->start_old_evacuations(); @@ -191,9 +191,6 @@ bool ShenandoahOldGC::collect(GCCause::Cause cause) { return false; } - // Prepare for old evacuations (actual evacuations will happen on subsequent young collects). - entry_old_evacuations(); - assert(!heap->is_concurrent_strong_root_in_progress(), "No evacuations during old gc."); vmop_entry_final_roots(); @@ -214,6 +211,10 @@ bool ShenandoahOldGC::collect(GCCause::Cause cause) { SpinPause(); } } + // Prepare for old evacuations (actual evacuations will happen on subsequent young collects). This cannot + // begin until after we have completed coalesce-and-fill. + start_old_evacuations(); + return true; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.hpp index d62f7d88371c3..a6ad266c21856 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.hpp @@ -42,7 +42,7 @@ class ShenandoahOldGC : public ShenandoahConcurrentGC { private: ShenandoahHeapRegion** _coalesce_and_fill_region_array; - void entry_old_evacuations(); + void start_old_evacuations(); bool entry_coalesce_and_fill(); ShenandoahSharedFlag& _allow_preemption; bool op_coalesce_and_fill(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp index f033c4b16847a..25ff159efb005 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp @@ -504,7 +504,7 @@ ShenandoahScanRemembered::process_clusters(size_t first_cluster, ShenandoahHeap* heap = ShenandoahHeap::heap(); ShenandoahMarkingContext* ctx; - if (heap->doing_mixed_evacuations()) { + if (heap->doing_mixed_evacuations() || heap->is_concurrent_prep_for_mixed_evacuation_in_progress()) { ctx = heap->marking_context(); } else { ctx = nullptr; From c5af3b95aa307f0bf151e0e89d6bbd45d1356dcd Mon Sep 17 00:00:00 2001 From: William Kemper Date: Wed, 10 Nov 2021 19:29:34 +0000 Subject: [PATCH 092/254] Padding used for alignment must fit an object Reviewed-by: rkennke --- .../share/gc/shenandoah/shenandoahHeapRegion.inline.hpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp index fde058fca8ad2..69c1c594cabfb 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp @@ -46,6 +46,9 @@ HeapWord* ShenandoahHeapRegion::allocate_aligned(size_t size, ShenandoahAllocReq if (pointer_delta(end(), obj + unalignment_words) >= size) { if (unalignment_words > 0) { size_t pad_words = (alignment_in_bytes / HeapWordSize) - unalignment_words; + if (pad_words < ShenandoahHeap::min_fill_size()) { + pad_words += (alignment_in_bytes / HeapWordSize); + } ShenandoahHeap::fill_with_object(obj, pad_words); ShenandoahHeap::heap()->card_scan()->register_object(obj); obj += pad_words; @@ -57,7 +60,7 @@ HeapWord* ShenandoahHeapRegion::allocate_aligned(size_t size, ShenandoahAllocReq HeapWord* new_top = obj + size; set_top(new_top); assert(is_object_aligned(new_top), "new top breaks alignment: " PTR_FORMAT, p2i(new_top)); - assert(((uintptr_t) obj) % (alignment_in_bytes) == 0, "obj is not aligned: " PTR_FORMAT, p2i(obj)); + assert(is_aligned(obj, alignment_in_bytes), "obj is not aligned: " PTR_FORMAT, p2i(obj)); return obj; } else { From 641e665cb30e3296b2f8ee942c00d616c0ca9cd0 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Tue, 16 Nov 2021 21:42:58 +0000 Subject: [PATCH 093/254] Remove assert that plab allocation must succeed Reviewed-by: kdnilsen --- .../share/gc/shenandoah/shenandoahFreeSet.cpp | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index 308314cb9b640..fa92c1ffce123 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -211,8 +211,7 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah } if (size >= req.min_size()) { result = r->allocate_aligned(size, req, CardTable::card_size); - assert (result != NULL, "Allocation must succeed: free " SIZE_FORMAT ", actual " SIZE_FORMAT, usable_free, size); - if (free > usable_free) { + if (result != nullptr && free > usable_free) { // Account for the alignment padding size_t padding = (free - usable_free) * HeapWordSize; increase_used(padding); @@ -241,15 +240,15 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah assert(size % CardTable::card_size_in_words == 0, "PLAB size must be multiple of remembered set card size"); result = r->allocate_aligned(size, req, CardTable::card_size); - assert (result != NULL, "Allocation must succeed: free " SIZE_FORMAT ", actual " SIZE_FORMAT, usable_free, size); - - // Account for the alignment padding - size_t padding = (free - usable_free) * HeapWordSize; - increase_used(padding); - assert(r->affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION, "All PLABs reside in old-gen"); - _heap->old_generation()->increase_used(padding); - // For verification consistency, we need to report this padding to _heap - _heap->increase_used(padding); + if (result != nullptr) { + // Account for the alignment padding + size_t padding = (free - usable_free) * HeapWordSize; + increase_used(padding); + assert(r->affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION, "All PLABs reside in old-gen"); + _heap->old_generation()->increase_used(padding); + // For verification consistency, we need to report this padding to _heap + _heap->increase_used(padding); + } } } else { result = r->allocate(size, req); From 191dd79a7efad684898f43e534caa0dd8c211d7c Mon Sep 17 00:00:00 2001 From: William Kemper Date: Thu, 18 Nov 2021 21:21:50 +0000 Subject: [PATCH 094/254] Clear young generation reference to old mark queues when starting global collect Reviewed-by: rkennke --- src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp index 11963a9439baa..b4a22df2f1bc7 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp @@ -29,6 +29,7 @@ #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" #include "gc/shenandoah/shenandoahUtils.hpp" #include "gc/shenandoah/shenandoahVerifier.hpp" +#include "gc/shenandoah/shenandoahYoungGeneration.hpp" #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" const char* ShenandoahGlobalGeneration::name() const { @@ -64,6 +65,7 @@ void ShenandoahGlobalGeneration::set_concurrent_mark_in_progress(bool in_progres // any stale state in the old generation. heap->purge_old_satb_buffers(true /* abandon */); heap->old_generation()->cancel_marking(); + heap->young_generation()->set_old_gen_task_queues(nullptr); } heap->set_concurrent_young_mark_in_progress(in_progress); From 0dfb0a3eede0f8a20286ff0edf4d50349633bff5 Mon Sep 17 00:00:00 2001 From: David Alvarez Date: Mon, 29 Nov 2021 19:57:34 +0000 Subject: [PATCH 095/254] Split Generational Shenandoah memory pools Reviewed-by: rkennke --- .../share/gc/shenandoah/shenandoahHeap.cpp | 26 +++++-- .../share/gc/shenandoah/shenandoahHeap.hpp | 3 + .../gc/shenandoah/shenandoahMemoryPool.cpp | 72 ++++++++++++++++++- .../gc/shenandoah/shenandoahMemoryPool.hpp | 33 +++++++-- 4 files changed, 122 insertions(+), 12 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 53d2370963f10..340d54a4e8922 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -525,6 +525,8 @@ ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) : _phase_timings(NULL), _monitoring_support(NULL), _memory_pool(NULL), + _young_gen_memory_pool(NULL), + _old_gen_memory_pool(NULL), _stw_memory_manager("Shenandoah Pauses", "end of GC pause"), _cycle_memory_manager("Shenandoah Cycles", "end of GC cycle"), _gc_timer(new (ResourceObj::C_HEAP, mtGC) ConcurrentGCTimer()), @@ -2604,9 +2606,18 @@ bool ShenandoahHeap::should_inject_alloc_failure() { } void ShenandoahHeap::initialize_serviceability() { - _memory_pool = new ShenandoahMemoryPool(this); - _cycle_memory_manager.add_pool(_memory_pool); - _stw_memory_manager.add_pool(_memory_pool); + if (mode()->is_generational()) { + _young_gen_memory_pool = new ShenandoahYoungGenMemoryPool(this); + _old_gen_memory_pool = new ShenandoahOldGenMemoryPool(this); + _cycle_memory_manager.add_pool(_young_gen_memory_pool); + _cycle_memory_manager.add_pool(_old_gen_memory_pool); + _stw_memory_manager.add_pool(_young_gen_memory_pool); + _stw_memory_manager.add_pool(_old_gen_memory_pool); + } else { + _memory_pool = new ShenandoahMemoryPool(this); + _cycle_memory_manager.add_pool(_memory_pool); + _stw_memory_manager.add_pool(_memory_pool); + } } GrowableArray ShenandoahHeap::memory_managers() { @@ -2618,12 +2629,17 @@ GrowableArray ShenandoahHeap::memory_managers() { GrowableArray ShenandoahHeap::memory_pools() { GrowableArray memory_pools(1); - memory_pools.append(_memory_pool); + if (mode()->is_generational()) { + memory_pools.append(_young_gen_memory_pool); + memory_pools.append(_old_gen_memory_pool); + } else { + memory_pools.append(_memory_pool); + } return memory_pools; } MemoryUsage ShenandoahHeap::memory_usage() { - return _memory_pool->get_memory_usage(); + return MemoryUsage(_initial_size, used(), committed(), max_capacity()); } ShenandoahRegionIterator::ShenandoahRegionIterator() : diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index daaeb5bc4be1e..b62cde156bf29 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -476,6 +476,9 @@ class ShenandoahHeap : public CollectedHeap { private: ShenandoahMonitoringSupport* _monitoring_support; MemoryPool* _memory_pool; + MemoryPool* _young_gen_memory_pool; + MemoryPool* _old_gen_memory_pool; + GCMemoryManager _stw_memory_manager; GCMemoryManager _cycle_memory_manager; ConcurrentGCTimer* _gc_timer; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMemoryPool.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMemoryPool.cpp index 339446e12e9f4..6821751412034 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMemoryPool.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMemoryPool.cpp @@ -24,14 +24,28 @@ #include "precompiled.hpp" #include "gc/shenandoah/shenandoahMemoryPool.hpp" +#include "gc/shenandoah/shenandoahYoungGeneration.hpp" +#include "gc/shenandoah/shenandoahOldGeneration.hpp" -ShenandoahMemoryPool::ShenandoahMemoryPool(ShenandoahHeap* heap) : - CollectedMemoryPool("Shenandoah", +ShenandoahMemoryPool::ShenandoahMemoryPool(ShenandoahHeap* heap, + const char* name) : + CollectedMemoryPool(name, heap->initial_capacity(), heap->max_capacity(), true /* support_usage_threshold */), _heap(heap) {} +ShenandoahMemoryPool::ShenandoahMemoryPool(ShenandoahHeap* heap, + const char* name, + size_t initial_capacity, + size_t max_capacity) : + CollectedMemoryPool(name, + initial_capacity, + max_capacity, + true /* support_usage_threshold */), + _heap(heap) {} + + MemoryUsage ShenandoahMemoryPool::get_memory_usage() { size_t initial = initial_size(); size_t max = max_size(); @@ -51,3 +65,57 @@ MemoryUsage ShenandoahMemoryPool::get_memory_usage() { return MemoryUsage(initial, used, committed, max); } + +size_t ShenandoahMemoryPool::used_in_bytes() { + return _heap->used(); +} + +size_t ShenandoahMemoryPool::max_size() const { + return _heap->max_capacity(); +} + +ShenandoahYoungGenMemoryPool::ShenandoahYoungGenMemoryPool(ShenandoahHeap* heap) : + ShenandoahMemoryPool(heap, + "Shenandoah Young Gen", + 0, + heap->max_capacity()) { } + +MemoryUsage ShenandoahYoungGenMemoryPool::get_memory_usage() { + size_t initial = initial_size(); + size_t max = max_size(); + size_t used = used_in_bytes(); + size_t committed = _heap->young_generation()->used_regions_size(); + + return MemoryUsage(initial, used, committed, max); +} + +size_t ShenandoahYoungGenMemoryPool::used_in_bytes() { + return _heap->young_generation()->used(); +} + +size_t ShenandoahYoungGenMemoryPool::max_size() const { + return _heap->young_generation()->max_capacity(); +} + +ShenandoahOldGenMemoryPool::ShenandoahOldGenMemoryPool(ShenandoahHeap* heap) : + ShenandoahMemoryPool(heap, + "Shenandoah Old Gen", + 0, + heap->max_capacity()) { } + +MemoryUsage ShenandoahOldGenMemoryPool::get_memory_usage() { + size_t initial = initial_size(); + size_t max = max_size(); + size_t used = used_in_bytes(); + size_t committed = _heap->old_generation()->used_regions_size(); + + return MemoryUsage(initial, used, committed, max); +} + +size_t ShenandoahOldGenMemoryPool::used_in_bytes() { + return _heap->old_generation()->used(); +} + +size_t ShenandoahOldGenMemoryPool::max_size() const { + return _heap->old_generation()->max_capacity(); +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMemoryPool.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMemoryPool.hpp index 2149213afa85e..29318ad249955 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMemoryPool.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMemoryPool.hpp @@ -32,14 +32,37 @@ #endif class ShenandoahMemoryPool : public CollectedMemoryPool { -private: +protected: ShenandoahHeap* _heap; public: - ShenandoahMemoryPool(ShenandoahHeap* pool); - MemoryUsage get_memory_usage(); - size_t used_in_bytes() { return _heap->used(); } - size_t max_size() const { return _heap->max_capacity(); } + ShenandoahMemoryPool(ShenandoahHeap* pool, + const char* name = "Shenandoah"); + virtual MemoryUsage get_memory_usage(); + virtual size_t used_in_bytes(); + virtual size_t max_size() const; + +protected: + ShenandoahMemoryPool(ShenandoahHeap* pool, + const char* name, + size_t initial_capacity, + size_t max_capacity); +}; + +class ShenandoahYoungGenMemoryPool : public ShenandoahMemoryPool { +public: + ShenandoahYoungGenMemoryPool(ShenandoahHeap* pool); + MemoryUsage get_memory_usage() override; + size_t used_in_bytes() override; + size_t max_size() const override; +}; + +class ShenandoahOldGenMemoryPool : public ShenandoahMemoryPool { +public: + ShenandoahOldGenMemoryPool(ShenandoahHeap* pool); + MemoryUsage get_memory_usage() override; + size_t used_in_bytes() override; + size_t max_size() const override; }; #endif // SHARE_GC_SHENANDOAH_SHENANDOAHMEMORYPOOL_HPP From da7bbab04e5f67aa33949dac60ddf16c72c073b8 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Tue, 30 Nov 2021 17:18:03 +0000 Subject: [PATCH 096/254] Add missing override keyword to fix MacOS (clang) builds Reviewed-by: kdnilsen --- .../heuristics/shenandoahOldHeuristics.hpp | 2 +- .../shenandoah/shenandoahGlobalGeneration.hpp | 24 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp index 2dee836d8a2bd..57823881196fc 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp @@ -80,7 +80,7 @@ class ShenandoahOldHeuristics : public ShenandoahHeuristics { ShenandoahOldHeuristics(ShenandoahGeneration* generation, ShenandoahHeuristics* trigger_heuristic); // Return true iff chosen collection set includes at least one old-gen HeapRegion. - virtual bool choose_collection_set(ShenandoahCollectionSet* collection_set, ShenandoahOldHeuristics* old_heuristics); + virtual bool choose_collection_set(ShenandoahCollectionSet* collection_set, ShenandoahOldHeuristics* old_heuristics) override; // Return true iff the collection set is primed with at least one old-gen region. bool prime_collection_set(ShenandoahCollectionSet* set); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp index cd4e50ddfd026..167b95ed7b3ad 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp @@ -34,28 +34,28 @@ class ShenandoahGlobalGeneration : public ShenandoahGeneration { : ShenandoahGeneration(GLOBAL, max_queues, 0, 0) { } public: - virtual const char* name() const; + virtual const char* name() const override; - virtual size_t max_capacity() const; - virtual size_t soft_max_capacity() const; - virtual size_t used_regions_size() const; - virtual size_t used() const; - virtual size_t available() const; + virtual size_t max_capacity() const override; + virtual size_t soft_max_capacity() const override; + virtual size_t used_regions_size() const override; + virtual size_t used() const override; + virtual size_t available() const override; - virtual void prepare_gc(bool do_old_gc_bootstrap); + virtual void prepare_gc(bool do_old_gc_bootstrap) override; - virtual void set_concurrent_mark_in_progress(bool in_progress); + virtual void set_concurrent_mark_in_progress(bool in_progress) override; - bool contains(ShenandoahHeapRegion* region) const; + bool contains(ShenandoahHeapRegion* region) const override; bool contains(oop obj) const override { return true; } - void parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl); + void parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl) override; - void heap_region_iterate(ShenandoahHeapRegionClosure* cl); + void heap_region_iterate(ShenandoahHeapRegionClosure* cl) override; protected: - bool is_concurrent_mark_in_progress(); + bool is_concurrent_mark_in_progress() override; }; #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHGLOBALGENERATION_HPP From 58064aa5f604fb07d66eb0b3fe673da377fa6b17 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Mon, 6 Dec 2021 20:15:11 +0000 Subject: [PATCH 097/254] Only verify last object start for marked objects Reviewed-by: kdnilsen --- .../gc/shenandoah/shenandoahScanRemembered.inline.hpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp index 25ff159efb005..66d8c2bd7bd62 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp @@ -438,18 +438,20 @@ ShenandoahScanRemembered::verify_registration(HeapWord* address, // should represent this object. Otherwise, last_offset is a don't care. ShenandoahHeapRegion* region = heap->heap_region_containing(base_addr + offset); HeapWord* tams = ctx->top_at_mark_start(region); + oop last_obj = nullptr; do { - prev_offset = offset; oop obj = cast_to_oop(base_addr + offset); if (ctx->is_marked(obj)) { + prev_offset = offset; offset += obj->size(); + last_obj = obj; } else { offset = ctx->get_next_marked_addr(base_addr + offset, tams) - base_addr; // offset will be zero if no objects are marked in this card. } } while (offset > 0 && offset < max_offset); - oop last_obj = cast_to_oop(base_addr + prev_offset); - if (prev_offset + last_obj->size() >= max_offset) { + if (last_obj != nullptr && prev_offset + last_obj->size() >= max_offset) { + // last marked object extends beyond end of card if (_scc->get_last_start(index) != prev_offset) { return false; } From 3e532346c1f52d7fc89f8143246920ce0765215d Mon Sep 17 00:00:00 2001 From: William Kemper Date: Wed, 12 Jan 2022 17:20:56 +0000 Subject: [PATCH 098/254] Reset top bitmap when region is immediately made into trash Reviewed-by: kdnilsen --- src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp | 4 +--- src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.cpp | 3 +-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index 505edf23c2492..3b10270a896a2 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -285,9 +285,7 @@ void ShenandoahHeapRegion::make_trash_immediate() { // On this path, we know there are no marked objects in the region, // tell marking context about it to bypass bitmap resets. assert(ShenandoahHeap::heap()->active_generation()->is_mark_complete(), "Marking should be complete here."); - // Leave top_bitmap alone. If it is greater than bottom(), then we still need to clear between bottom() and top_bitmap() - // when this FREE region is repurposed for YOUNG or OLD. - // ShenandoahHeap::heap()->marking_context()->reset_top_bitmap(this); + ShenandoahHeap::heap()->marking_context()->reset_top_bitmap(this); } void ShenandoahHeapRegion::make_empty() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.cpp index f50ad37a4e9e9..6afa6ca0689dd 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.cpp @@ -67,8 +67,7 @@ void ShenandoahMarkingContext::initialize_top_at_mark_start(ShenandoahHeapRegion HeapWord *bottom = r->bottom(); _top_at_mark_starts_base[idx] = bottom; - // Arrange that the first time we use this bitmap, we clean from bottom to end. - _top_bitmaps[idx] = r->end(); + _top_bitmaps[idx] = bottom; log_debug(gc)("SMC:initialize_top_at_mark_start for Region " SIZE_FORMAT ", TAMS: " PTR_FORMAT ", TopOfBitMap: " PTR_FORMAT, r->index(), p2i(bottom), p2i(r->end())); From 7d0a45c3fb58db5be86a52feb320a62af69e3cb5 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Thu, 27 Jan 2022 18:23:17 +0000 Subject: [PATCH 099/254] Degenerate to marking phase if cancellation detected after final mark Reviewed-by: rkennke --- src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index d0b175584b5b4..f0460cd27b938 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -123,6 +123,8 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) { // Complete marking under STW, and start evacuation vmop_entry_final_mark(); + check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark); + // Concurrent stack processing if (heap->is_evacuation_in_progress()) { entry_thread_roots(); From 915bea80d628b8dd7e448cfdacdb378261f8e07c Mon Sep 17 00:00:00 2001 From: William Kemper Date: Thu, 27 Jan 2022 19:13:44 +0000 Subject: [PATCH 100/254] Update card table when cycle degenerates during root scan Reviewed-by: kdnilsen --- .../gc/shenandoah/shenandoahDegeneratedGC.cpp | 9 ++++++ .../gc/shenandoah/shenandoahGeneration.cpp | 32 +++++++++++++++++++ .../gc/shenandoah/shenandoahGeneration.hpp | 3 ++ .../shenandoah/shenandoahScanRemembered.hpp | 15 +++++++++ 4 files changed, 59 insertions(+) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp index e8f6a816c3730..565a3490a6ee6 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp @@ -128,6 +128,15 @@ void ShenandoahDegenGC::op_degenerated() { heap->cancel_concurrent_mark(); } + if (_degen_point == ShenandoahDegenPoint::_degenerated_roots) { + // We only need this if the concurrent cycle has already swapped the card tables. + // Marking will use the 'read' table, but interesting pointers may have been + // recorded in the 'write' table in the time between the cancelled concurrent cycle + // and this degenerated cycle. These pointers need to be included the 'read' table + // used to scan the remembered set during the STW mark which follows here. + _generation->merge_write_table(); + } + op_reset(); // STW mark diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index 27473528f4dba..9818e03f01176 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -72,6 +72,24 @@ class ShenandoahResetBitmapTask : public ShenandoahHeapRegionClosure { bool is_thread_safe() { return true; } }; +class ShenandoahMergeWriteTable: public ShenandoahHeapRegionClosure { + private: + ShenandoahHeap* _heap; + RememberedScanner* _scanner; + public: + ShenandoahMergeWriteTable() : _heap(ShenandoahHeap::heap()), _scanner(_heap->card_scan()) {} + + virtual void heap_region_do(ShenandoahHeapRegion* r) override { + if (r->is_old()) { + _scanner->merge_write_table(r->bottom(), ShenandoahHeapRegion::region_size_words()); + } + } + + virtual bool is_thread_safe() override { + return true; + } +}; + class ShenandoahSquirrelAwayCardTable: public ShenandoahHeapRegionClosure { private: ShenandoahHeap* _heap; @@ -171,6 +189,20 @@ void ShenandoahGeneration::swap_remembered_set() { heap->old_generation()->parallel_heap_region_iterate(&task); } +// If a concurrent cycle fails _after_ the card table has been swapped we need to update the read card +// table with any writes that have occurred during the transition to the degenerated cycle. Without this, +// newly created objects which are only referenced by old objects could be lost when the remembered set +// is scanned during the degenerated mark. +void ShenandoahGeneration::merge_write_table() { + // This should only happen for degenerated cycles + ShenandoahHeap* heap = ShenandoahHeap::heap(); + heap->assert_gc_workers(heap->workers()->active_workers()); + shenandoah_assert_safepoint(); + + ShenandoahMergeWriteTable task; + heap->old_generation()->parallel_heap_region_iterate(&task); +} + void ShenandoahGeneration::prepare_gc(bool do_old_gc_bootstrap) { // Reset mark bitmap for this generation (typically young) reset_mark_bitmap(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index 902273bbf4d5b..5773460f09f2e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -90,6 +90,9 @@ class ShenandoahGeneration : public CHeapObj { // Used by concurrent and degenerated GC to reset remembered set. void swap_remembered_set(); + // Update the read cards with the state of the write table (write table is not cleared). + void merge_write_table(); + // Used by concurrent and degenerated GC to reset regions. virtual void prepare_gc(bool do_old_gc_bootstrap); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp index 70f9685042ed1..3b74f440b0c68 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp @@ -283,6 +283,19 @@ class ShenandoahDirectCardMarkRememberedSet: public CHeapObj { // Not currently used because mutator write barrier does not honor changes to the location of card table. void swap_remset() { _card_table->swap_card_tables(); } + void merge_write_table(HeapWord* start, size_t word_count) { + size_t card_index = card_index_for_addr(start); + size_t num_cards = word_count / CardTable::card_size_in_words; + size_t iterations = num_cards / (sizeof (intptr_t) / sizeof (CardTable::CardValue)); + intptr_t* read_table_ptr = (intptr_t*) &(_card_table->read_byte_map())[card_index]; + intptr_t* write_table_ptr = (intptr_t*) &(_card_table->write_byte_map())[card_index]; + for (size_t i = 0; i < iterations; i++) { + intptr_t card_value = *write_table_ptr; + *read_table_ptr++ &= card_value; + write_table_ptr++; + } + } + HeapWord* whole_heap_base() { return _whole_heap_base; } HeapWord* whole_heap_end() { return _whole_heap_end; } @@ -916,6 +929,8 @@ class ShenandoahScanRemembered: public CHeapObj { void reset_remset(HeapWord* start, size_t word_count) { _rs->reset_remset(start, word_count); } + void merge_write_table(HeapWord* start, size_t word_count) { _rs->merge_write_table(start, word_count); } + // Called by GC thread after scanning old remembered set in order to prepare for next GC pass void clear_old_remset() { _rs->clear_old_remset(); } From e91c55ccd5885e23d4666ab915860a1f61558f94 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Tue, 1 Feb 2022 21:57:19 +0000 Subject: [PATCH 101/254] Simplify crossing map implementation Reviewed-by: wkemper --- .../shenandoah/shenandoahScanRemembered.hpp | 66 ++++++++----------- .../shenandoahScanRemembered.inline.hpp | 36 +++++----- 2 files changed, 46 insertions(+), 56 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp index 3b74f440b0c68..5bcbb1413c90f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp @@ -513,71 +513,65 @@ class ShenandoahCardCluster: public CHeapObj { static const size_t CardsPerCluster = 64; private: - // This bit is set iff at least one object starts within a - // particular card region. - static const uint16_t ObjectStartsInCardRegion = 0x8000; - static const uint16_t FirstStartBits = 0x003f; - static const uint16_t LastStartBits = 0x0fc0; - static const uint16_t FirstStartShift = 0; - static const uint16_t LastStartShift = 6; + typedef struct cross_map { uint8_t first; uint8_t last; } xmap; + typedef union crossing_info { uint16_t short_word; xmap offsets; } crossing_info; - uint16_t *object_starts; + // ObjectStartsInCardRegion bit is set within a crossing_info.offsets.start iff at least one object starts within + // a particular card region. We pack this bit into start byte under assumption that start byte is accessed less + // frequently that last byte. This is true when number of clean cards is greater than number of dirty cards. + static const uint16_t ObjectStartsInCardRegion = 0x80; + static const uint16_t FirstStartBits = 0x3f; + + crossing_info *object_starts; public: + // If we're setting first_start, assume the card has an object. inline void set_first_start(size_t card_index, uint8_t value) { - object_starts[card_index] &= ~FirstStartBits; - object_starts[card_index] |= (FirstStartBits & (value << FirstStartShift)); + object_starts[card_index].offsets.first = ObjectStartsInCardRegion | value; } inline void set_last_start(size_t card_index, uint8_t value) { - object_starts[card_index] &= ~LastStartBits; - object_starts[card_index] |= (LastStartBits & (value << LastStartShift)); + object_starts[card_index].offsets.last = value; } inline void set_has_object_bit(size_t card_index) { - object_starts[card_index] |= ObjectStartsInCardRegion; + object_starts[card_index].offsets.first |= ObjectStartsInCardRegion; } inline void clear_has_object_bit(size_t card_index) { - object_starts[card_index] &= ~ObjectStartsInCardRegion; + object_starts[card_index].offsets.first &= ~ObjectStartsInCardRegion; + } + + // Returns true iff an object is known to start within the card memory associated with card card_index. + inline bool has_object(size_t card_index) { + return (object_starts[card_index].offsets.first & ObjectStartsInCardRegion) != 0; } inline void clear_objects_in_range(HeapWord *addr, size_t num_words) { size_t card_index = _rs->card_index_for_addr(addr); size_t last_card_index = _rs->card_index_for_addr(addr + num_words - 1); while (card_index <= last_card_index) - object_starts[card_index++] = 0; + object_starts[card_index++].short_word = 0; } ShenandoahCardCluster(RememberedSet *rs) { _rs = rs; // TODO: We don't really need object_starts entries for every card entry. We only need these for // the card entries that correspond to old-gen memory. But for now, let's be quick and dirty. - object_starts = (uint16_t *) malloc(rs->total_cards() * sizeof(uint16_t)); - if (object_starts == NULL) + object_starts = (crossing_info *) malloc(rs->total_cards() * sizeof(crossing_info)); + if (object_starts == nullptr) fatal("Insufficient memory for initializing heap"); for (size_t i = 0; i < rs->total_cards(); i++) - object_starts[i] = 0; + object_starts[i].short_word = 0; } ~ShenandoahCardCluster() { - if (object_starts != NULL) + if (object_starts != nullptr) free(object_starts); + object_starts = nullptr; } - // There is one entry within the object_starts array for each - // card entry. The interpretation of the data contained within each - // object_starts entry is described below: - // - // Bits 0x003f: Value ranges from 0-63, which is multiplied by 8 - // to obtain the offset at which the first object - // beginning within this card region begins. - // Bits 0x0fc0: Value ranges from 0-63, which is multiplied by 8 to - // obtain the offset at which the last object beginning - // within this card region begins. - // Bits 0x8000: This bit is on if an object starts within this card - // region. - // Bits 0x7000: Reserved for future uses + // There is one entry within the object_starts array for each card entry. // // In the most recent implementation of ShenandoahScanRemembered::process_clusters(), // there is no need for the get_crossing_object_start() method function, so there is no @@ -635,7 +629,7 @@ class ShenandoahCardCluster: public CHeapObj { // 2. The cost can be deferred so that there is no urgency during // mutator copy-on-first-access promotion. Background GC // threads will update the object_starts array by post- - // processing the contents of retired GCLAB buffers. + // processing the contents of retired PLAB buffers. // 3. The bet is that these costs are paid relatively rarely // because: // a) Most objects die young and objects that die in young-gen @@ -821,12 +815,6 @@ class ShenandoahCardCluster: public CHeapObj { // the card number pertaining to a particular address and then uses the // card noumber for subsequent information lookups and stores. - // Returns true iff an object is known to start within the card memory - // associated with addr p. - // Returns true iff an object is known to start within the card memory - // associated with addr p. - bool has_object(size_t card_index); - // If has_object(card_index), this returns the word offset within this card // memory at which the first object begins. If !has_object(card_index), the // result is a don't care value. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp index 66d8c2bd7bd62..470f270d28a96 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp @@ -171,7 +171,7 @@ ShenandoahCardCluster::reset_object_range(HeapWord* from, HeapWor size_t num_cards = (to - from) / CardTable::card_size_in_words; for (size_t i = 0; i < num_cards; i++) { - object_starts[card_at_start + i] = 0; + object_starts[card_at_start + i].short_word = 0; } } @@ -181,6 +181,7 @@ template inline void ShenandoahCardCluster::register_object(HeapWord* address) { shenandoah_assert_heaplocked(); + register_object_wo_lock(address); } @@ -191,7 +192,7 @@ ShenandoahCardCluster::register_object_wo_lock(HeapWord* address) HeapWord *card_start_address = _rs->addr_for_card_index(card_at_start); uint8_t offset_in_card = address - card_start_address; - if ((object_starts[card_at_start] & ObjectStartsInCardRegion) == 0) { + if (!has_object(card_at_start)) { set_has_object_bit(card_at_start); set_first_start(card_at_start, offset_in_card); set_last_start(card_at_start, offset_in_card); @@ -212,11 +213,18 @@ ShenandoahCardCluster::coalesce_objects(HeapWord* address, size_t size_t card_at_end = card_at_start + ((address + length_in_words) - card_start_address) / CardTable::card_size_in_words; if (card_at_start == card_at_end) { - // No changes to object_starts array. Either: - // get_first_start(card_at_start) returns this coalesced object, - // or it returns an object that precedes the coalesced object. - // get_last_start(card_at_start) returns the object that immediately follows the coalesced object, - // or it returns an object that comes after the object immediately following the coalesced object. + // There are no changes to the get_first_start array. Either get_first_start(card_at_start) returns this coalesced object, + // or it returns an object that precedes the coalesced object. + if (card_start_address + get_last_start(card_at_start) < address + length_in_words) { + uint8_t coalesced_offset = static_cast(address - card_start_address); + // The object that used to be the last object starting within this card is being subsumed within the coalesced + // object. Since we always coalesce entire objects, this condition only occurs if the last object ends before or at + // the end of the card's memory range and there is no object following this object. In this case, adjust last_start + // to represent the start of the coalesced range. + set_last_start(card_at_start, coalesced_offset); + } + // Else, no changes to last_starts information. Either get_last_start(card_at_start) returns the object that immediately + // follows the coalesced object, or it returns an object that follows the object immediately following the coalesced object. } else { uint8_t coalesced_offset = static_cast(address - card_start_address); if (get_last_start(card_at_start) > coalesced_offset) { @@ -249,24 +257,18 @@ ShenandoahCardCluster::coalesce_objects(HeapWord* address, size_t } -template -inline bool -ShenandoahCardCluster::has_object(size_t card_index) { - return object_starts[card_index] & ObjectStartsInCardRegion; -} - template inline size_t ShenandoahCardCluster::get_first_start(size_t card_index) { - assert(object_starts[card_index] & ObjectStartsInCardRegion, "Can't get first start because no object starts here"); - return (object_starts[card_index] & FirstStartBits) >> FirstStartShift; + assert(has_object(card_index), "Can't get first start because no object starts here"); + return object_starts[card_index].offsets.first & FirstStartBits; } template inline size_t ShenandoahCardCluster::get_last_start(size_t card_index) { - assert(object_starts[card_index] & ObjectStartsInCardRegion, "Can't get last start because no objects starts here"); - return (object_starts[card_index] & LastStartBits) >> LastStartShift; + assert(has_object(card_index), "Can't get last start because no object starts here"); + return object_starts[card_index].offsets.last; } template From 006e14e4f83fadefad8d3f5353fe5e41db2c7028 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Tue, 1 Feb 2022 22:46:15 +0000 Subject: [PATCH 102/254] Coalesce and fill objects immediately after final mark Reviewed-by: kdnilsen --- .../share/gc/shenandoah/shenandoahConcurrentGC.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index f0460cd27b938..e3a574ce7189f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -125,6 +125,12 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) { check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark); + // Global marking has completed. We need to fill in any unmarked objects in the old generation + // so that subsequent remembered set scans will not walk pointers into reclaimed memory. + if (!heap->cancelled_gc() && heap->mode()->is_generational() && _generation->generation_mode() == GLOBAL) { + entry_global_coalesce_and_fill(); + } + // Concurrent stack processing if (heap->is_evacuation_in_progress()) { entry_thread_roots(); @@ -159,10 +165,6 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) { entry_strong_roots(); } - if (!heap->cancelled_gc() && heap->mode()->is_generational() && _generation->generation_mode() == GLOBAL) { - entry_global_coalesce_and_fill(); - } - // Continue the cycle with evacuation and optional update-refs. // This may be skipped if there is nothing to evacuate. // If so, evac_in_progress would be unset by collection set preparation code. From b233fe7c6cdbbfb721b5d3b1ef1e4e50a861faff Mon Sep 17 00:00:00 2001 From: William Kemper Date: Fri, 4 Feb 2022 16:58:43 +0000 Subject: [PATCH 103/254] Improve handling of cancellation after final mark Reviewed-by: kdnilsen --- .../share/gc/shenandoah/shenandoahConcurrentGC.cpp | 13 ++++++++++++- .../share/gc/shenandoah/shenandoahGeneration.hpp | 2 -- .../gc/shenandoah/shenandoahGlobalGeneration.hpp | 1 - .../share/gc/shenandoah/shenandoahOldGeneration.hpp | 2 +- .../gc/shenandoah/shenandoahYoungGeneration.hpp | 1 - 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index e3a574ce7189f..b0917b10589d5 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -123,7 +123,18 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) { // Complete marking under STW, and start evacuation vmop_entry_final_mark(); - check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark); + // If GC was cancelled before final mark, then the safepoint operation will do nothing + // and the concurrent mark will still be in progress. In this case it is safe to resume + // the degenerated cycle from the marking phase. On the other hand, if the GC is cancelled + // after final mark (but before this check), then the final mark safepoint operation + // will have finished the mark (setting concurrent mark in progress to false). Final mark + // will also have setup state (in concurrent stack processing) that will not be safe to + // resume from the marking phase in the degenerated cycle. That is, if the cancellation + // occurred after final mark, we must resume the degenerated cycle after the marking phase. + if (_generation->is_concurrent_mark_in_progress() && check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark)) { + assert(!heap->is_concurrent_weak_root_in_progress(), "Weak roots should not be in progress when concurrent mark is in progress"); + return false; + } // Global marking has completed. We need to fill in any unmarked objects in the old generation // so that subsequent remembered set scans will not walk pointers into reclaimed memory. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index 5773460f09f2e..54e8aba0bf3ff 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -142,8 +142,6 @@ class ShenandoahGeneration : public CHeapObj { void increase_used(size_t bytes); void decrease_used(size_t bytes); -protected: - virtual bool is_concurrent_mark_in_progress() = 0; private: diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp index 167b95ed7b3ad..24b4b81fd424e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp @@ -54,7 +54,6 @@ class ShenandoahGlobalGeneration : public ShenandoahGeneration { void heap_region_iterate(ShenandoahHeapRegionClosure* cl) override; - protected: bool is_concurrent_mark_in_progress() override; }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp index 9b146051f895e..1e61d14641d0c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp @@ -69,7 +69,7 @@ class ShenandoahOldGeneration : public ShenandoahGeneration { // object at the barrier, but we reject this approach because it is likely // the performance impact would be too severe. void purge_satb_buffers(bool abandon); - protected: + bool is_concurrent_mark_in_progress() override; }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp index 5b1664cd089fd..ab200bd0e5417 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp @@ -54,7 +54,6 @@ class ShenandoahYoungGeneration : public ShenandoahGeneration { void reserve_task_queues(uint workers) override; - protected: bool is_concurrent_mark_in_progress() override; }; From c4bafaebea4f85e35c025b2205d7978213ce66e1 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Tue, 8 Feb 2022 17:47:47 +0000 Subject: [PATCH 104/254] Coalesce and fill dead objects after class unloading Reviewed-by: kdnilsen --- .../share/gc/shenandoah/shenandoahConcurrentGC.cpp | 12 ++++++------ .../share/gc/shenandoah/shenandoahDegeneratedGC.cpp | 3 ++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index b0917b10589d5..d989970e96ece 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -136,12 +136,6 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) { return false; } - // Global marking has completed. We need to fill in any unmarked objects in the old generation - // so that subsequent remembered set scans will not walk pointers into reclaimed memory. - if (!heap->cancelled_gc() && heap->mode()->is_generational() && _generation->generation_mode() == GLOBAL) { - entry_global_coalesce_and_fill(); - } - // Concurrent stack processing if (heap->is_evacuation_in_progress()) { entry_thread_roots(); @@ -176,6 +170,12 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) { entry_strong_roots(); } + // Global marking has completed. We need to fill in any unmarked objects in the old generation + // so that subsequent remembered set scans will not walk pointers into reclaimed memory. + if (!heap->cancelled_gc() && heap->mode()->is_generational() && _generation->generation_mode() == GLOBAL) { + entry_global_coalesce_and_fill(); + } + // Continue the cycle with evacuation and optional update-refs. // This may be skipped if there is nothing to evacuate. // If so, evac_in_progress would be unset by collection set preparation code. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp index 565a3490a6ee6..c90ebd21a70ae 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp @@ -154,11 +154,12 @@ void ShenandoahDegenGC::op_degenerated() { op_cleanup_early(); + case _degenerated_evac: + if (heap->mode()->is_generational() && _generation->generation_mode() == GLOBAL) { op_global_coalesce_and_fill(); } - case _degenerated_evac: // If heuristics thinks we should do the cycle, this flag would be set, // and we can do evacuation. Otherwise, it would be the shortcut cycle. if (heap->is_evacuation_in_progress()) { From e1d15c1f23d51933f89be2224221af3b5482e87c Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Tue, 8 Feb 2022 17:50:31 +0000 Subject: [PATCH 105/254] Borrow from old gen Reviewed-by: wkemper --- .../shenandoahAdaptiveHeuristics.cpp | 141 +++++++++++-- .../shenandoahAggressiveHeuristics.cpp | 4 + .../heuristics/shenandoahHeuristics.cpp | 182 ++++++++++++++++- .../heuristics/shenandoahHeuristics.hpp | 11 + .../heuristics/shenandoahOldHeuristics.cpp | 116 +++++------ .../shenandoahPassiveHeuristics.cpp | 2 +- .../mode/shenandoahGenerationalMode.cpp | 13 ++ .../mode/shenandoahGenerationalMode.hpp | 1 + .../gc/shenandoah/shenandoahCollectionSet.cpp | 14 ++ .../gc/shenandoah/shenandoahCollectionSet.hpp | 28 +++ .../shenandoahCollectionSet.inline.hpp | 28 +++ .../shenandoah/shenandoahCollectorPolicy.cpp | 1 + .../gc/shenandoah/shenandoahConcurrentGC.cpp | 82 +++++++- .../gc/shenandoah/shenandoahConcurrentGC.hpp | 2 +- .../gc/shenandoah/shenandoahDegeneratedGC.cpp | 19 ++ .../share/gc/shenandoah/shenandoahFreeSet.cpp | 53 +++-- .../share/gc/shenandoah/shenandoahFreeSet.hpp | 9 +- .../share/gc/shenandoah/shenandoahFullGC.cpp | 54 ++++- .../gc/shenandoah/shenandoahGeneration.cpp | 138 ++++++++++++- .../gc/shenandoah/shenandoahGeneration.hpp | 17 +- .../share/gc/shenandoah/shenandoahHeap.cpp | 192 ++++++++++++++---- .../share/gc/shenandoah/shenandoahHeap.hpp | 103 +++++++++- .../gc/shenandoah/shenandoahHeap.inline.hpp | 126 +++++++++++- .../gc/shenandoah/shenandoahHeapRegion.cpp | 2 +- .../gc/shenandoah/shenandoahHeapRegion.hpp | 3 +- .../shenandoahHeapRegion.inline.hpp | 4 +- .../gc/shenandoah/shenandoahMarkClosures.cpp | 15 -- .../share/gc/shenandoah/shenandoahOldGC.cpp | 2 +- .../shenandoah/shenandoahThreadLocalData.hpp | 51 +++++ .../share/gc/shenandoah/shenandoahUtils.cpp | 8 + .../gc/shenandoah/shenandoahVMOperations.cpp | 18 ++ .../gc/shenandoah/shenandoahVMOperations.hpp | 5 +- .../gc/shenandoah/shenandoahVerifier.cpp | 15 +- .../shenandoah/shenandoahYoungGeneration.cpp | 7 + .../shenandoah/shenandoahYoungGeneration.hpp | 4 + .../gc/shenandoah/shenandoah_globals.hpp | 107 +++++++--- 36 files changed, 1370 insertions(+), 207 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp index 35c5786b3b84b..c9886ffb9abd5 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp @@ -29,6 +29,7 @@ #include "gc/shenandoah/shenandoahFreeSet.hpp" #include "gc/shenandoah/shenandoahGeneration.hpp" #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" +#include "gc/shenandoah/shenandoahYoungGeneration.hpp" #include "logging/log.hpp" #include "logging/logTag.hpp" #include "utilities/quickSort.hpp" @@ -84,38 +85,62 @@ void ShenandoahAdaptiveHeuristics::choose_collection_set_from_regiondata(Shenand // we hit max_cset. When max_cset is hit, we terminate the cset selection. Note that in this scheme, // ShenandoahGarbageThreshold is the soft threshold which would be ignored until min_garbage is hit. - size_t capacity = _generation->soft_max_capacity(); - size_t max_cset = (size_t)((1.0 * capacity / 100 * ShenandoahEvacReserve) / ShenandoahEvacWaste); - size_t free_target = (capacity / 100 * ShenandoahMinFreeThreshold) + max_cset; - size_t min_garbage = (free_target > actual_free ? (free_target - actual_free) : 0); + size_t max_cset = (ShenandoahHeap::heap()->get_young_evac_reserve() / ShenandoahEvacWaste); + size_t capacity = ShenandoahHeap::heap()->young_generation()->soft_max_capacity(); - log_info(gc, ergo)("Adaptive CSet Selection. Target Free: " SIZE_FORMAT "%s, Actual Free: " - SIZE_FORMAT "%s, Max CSet: " SIZE_FORMAT "%s, Min Garbage: " SIZE_FORMAT "%s", - byte_size_in_proper_unit(free_target), proper_unit_for_byte_size(free_target), - byte_size_in_proper_unit(actual_free), proper_unit_for_byte_size(actual_free), + // As currently implemented, we are not enforcing that new_garbage > min_garbage + // size_t free_target = (capacity / 100) * ShenandoahMinFreeThreshold + max_cset; + // size_t min_garbage = (free_target > actual_free ? (free_target - actual_free) : 0); + + log_info(gc, ergo)("Adaptive CSet Selection. Max CSet: " SIZE_FORMAT "%s, Actual Free: " SIZE_FORMAT "%s.", byte_size_in_proper_unit(max_cset), proper_unit_for_byte_size(max_cset), - byte_size_in_proper_unit(min_garbage), proper_unit_for_byte_size(min_garbage)); + byte_size_in_proper_unit(actual_free), proper_unit_for_byte_size(actual_free)); // Better select garbage-first regions QuickSort::sort(data, (int)size, compare_by_garbage, false); size_t cur_cset = 0; - size_t cur_garbage = 0; + // size_t cur_garbage = 0; + + // In generational mode, the sort order within the data array is not strictly descending amounts of garbage. In + // particular, regions that have reached tenure age will be sorted into this array before younger regions that contain + // more garbage. This represents one of the reasons why we keep looking at regions even after we decide, for example, + // to exclude one of the regions because it might require evacuation of too much live data. for (size_t idx = 0; idx < size; idx++) { ShenandoahHeapRegion* r = data[idx]._region; + size_t biased_garbage = data[idx]._garbage; size_t new_cset = cur_cset + r->get_live_data_bytes(); - size_t new_garbage = cur_garbage + r->garbage(); - - if (new_cset > max_cset) { - break; - } - if ((new_garbage < min_garbage) || (r->garbage() > garbage_threshold)) { + // As currently implemented, we are not enforcing that new_garbage > min_garbage + // size_t new_garbage = cur_garbage + r->garbage(); + + // Note that live data bytes within a region is not the same as heap_region_size - garbage. This is because + // each region contains a combination of used memory (which is garbage plus live) and unused memory, which has not + // yet been allocated. It may be the case that the region on this iteration has too much live data to be added to + // the collection set while one or more regions seen on subsequent iterations of this loop can be added to the collection + // set because they have smaller live memory, even though they also have smaller garbage (and necessarily a larger + // amount of unallocated memory). + + // BANDAID: In an earlier version of this code, this was written: + // if ((new_cset <= max_cset) && ((new_garbage < min_garbage) || (r->garbage() > garbage_threshold))) + // The problem with the original code is that in some cases the collection set would include hundreds of regions, + // each with less than 100 bytes of garbage. Evacuating these regions is counterproductive. + + // TODO: Think about changing the description and defaults for ShenandoahGarbageThreshold and ShenandoahMinFreeThreshold. + // If "customers" want to evacuate regions with smaller amounts of garbage contained therein, they should specify a lower + // value of ShenandoahGarbageThreshold. As implemented currently, we may experience back-to-back collections if there is + // not enough memory to be reclaimed. Let's not let pursuit of min_garbage drive us to make poor decisions. Maybe we + // want yet another global parameter to allow a region to be placed into the collection set if + // (((new_garbage < min_garbage) && (r->garbage() > ShenandoahSmallerGarbageThreshold)) || (r->garbage() > garbage_threshold)) + + if ((new_cset <= max_cset) && ((r->garbage() > garbage_threshold) || (r->age() >= InitialTenuringThreshold))) { cset->add_region(r); cur_cset = new_cset; - cur_garbage = new_garbage; + // cur_garbage = new_garbage; + } else if (biased_garbage == 0) { + break; } } } @@ -215,9 +240,6 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { size_t min_threshold = capacity / 100 * ShenandoahMinFreeThreshold; - log_debug(gc)(" available adjusted to: " SIZE_FORMAT ", min_threshold: " SIZE_FORMAT ", ShenandoahMinFreeThreshold: " SIZE_FORMAT, - available, min_threshold, ShenandoahMinFreeThreshold); - if (available < min_threshold) { log_info(gc)("Trigger (%s): Free (" SIZE_FORMAT "%s) is below minimum threshold (" SIZE_FORMAT "%s)", _generation->name(), @@ -244,18 +266,96 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { // 2. Accumulated penalties from Degenerated and Full GC size_t allocation_headroom = available; + // ShenandoahAllocSpikeFactor is the percentage of capacity that we endeavor to assure to be free at the end of the GC + // cycle. + // TODO: Correct the representation of this quantity + // (and dive deeper into _gc_time_penalties as this may also need to be corrected) + // + // Allocation spikes are a characteristic of both the application ahd the JVM configuration. On the JVM command line, + // the application developer may want to supply a hint of the nature of spikes that are inherent in the application + // workload, and this information would normally be independent of heap size (not a percentage thereof). On the + // other hand, some allocation spikes are correlated with JVM configuration. For example, there are allocation + // spikes at the starts of concurrent marking and evacuation to refresh all local allocation buffers. The nature + // of these spikes is determined by LAB min and max sizes and numbers of threads, but also on frequency of GC passes, + // and on "periodic" behavior of these threads If GC frequency is much higher than the periodic trigger for mutator + // threads, then many of the mutator threads may be able to "sit out" of most GC passes. Though the thread's stack + // must be scanned, the thread does not need to refresh its LABs if it sits idle throughout the duration of the GC + // pass. The best prediction for this aspect of spikes in allocation patterns is probably recent past history. + // + // Rationale: + // The idea is that there is an average allocation rate and there are occassional abnormal bursts (or spikes) of + // allocations that exceed the average allocation rate. What do these spikes look like? + // + // 1. At certain phase changes, we may discard large amounts of data and replace it with large numbers of newly + // allocated objects. This "spike" looks more like a phase change. We were in steady state at M bytes/sec + // allocation rate and now we're in a "reinitialization phase" that looks like N bytes/sec. We need the "spike" + // accomodation to give us enough runway to recalibrate our "average allocation rate". + // + // 2. The typical workload changes. "Suddenly", our typical workload of N TPS increases to N+delta TPS. This means + // our average allocation rate needs to be adjusted. Once again, we need the "spike" accomodation to give us + // enough runway to recalibrate our "average allocation rate". + // + // 3. Though there is an "average" allocation rate, a given workload's demand for allocation may be very bursty. We + // allocate a bunch of LABs during the 5 ms that follow completion of a GC, then we perform no more allocations for + // the next 150 ms. It seems we want the "spike" to represent the maximum divergence from average within the + // period of time between consecutive evaluation of the should_start_gc() service. Here's the thinking: + // + // a) Between now and the next time I ask whether should_start_gc(), we might experience a spike representing + // the anticipated burst of allocations. If that would put us over budget, then we should start GC immediately. + // b) Between now and the anticipated depletion of allocation pool, there may be two or more bursts of allocations. + // If there are more than one of these bursts, we can "approximate" that these will be separated by spans of + // time with very little or no allocations so the "average" allocation rate should be a suitable approximation + // of how this will behave. + // + // For cases 1 and 2, we need to "quickly" recalibrate the average allocation rate whenever we detect a change + // in operation mode. We want some way to decide that the average rate has changed. Make average allocation rate + // computations an independent effort. + size_t spike_headroom = capacity / 100 * ShenandoahAllocSpikeFactor; size_t penalties = capacity / 100 * _gc_time_penalties; + // TODO: Account for inherent delays in responding to GC triggers + // 1. It has been observed that delays of 200 ms or greater are common between the moment we return true from should_start_gc() + // and the moment at which we begin execution of the concurrent reset phase. Add this time into the calculation of + // avg_cycle_time below. (What is "this time"? Perhaps we should remember recent history of this delay for the + // running workload and use the maximum delay recently seen for "this time".) + // 2. The frequency of inquiries to should_start_gc() is adaptive, ranging between ShenandoahControlIntervalMin and + // ShenandoahControlIntervalMax. The current control interval (or the max control interval) should also be added into + // the calculation of avg_cycle_time below. + allocation_headroom -= MIN2(allocation_headroom, spike_headroom); allocation_headroom -= MIN2(allocation_headroom, penalties); double avg_cycle_time = _gc_time_history->davg() + (_margin_of_error_sd * _gc_time_history->dsd()); + + size_t last_live_memory = get_last_live_memory(); + size_t penultimate_live_memory = get_penultimate_live_memory(); + double original_cycle_time = avg_cycle_time; + if ((penultimate_live_memory < last_live_memory) && (penultimate_live_memory != 0)) { + // If the live-memory size is growing, our estimates of cycle time are based on lighter workload, so adjust. + // TODO: Be more precise about how to scale when live memory is growing. Existing code is a very rough approximation + // tuned with very limited workload observations. + avg_cycle_time = (avg_cycle_time * 2 * last_live_memory) / penultimate_live_memory; + } else { + int degen_cycles = degenerated_cycles_in_a_row(); + if (degen_cycles > 0) { + // If we've degenerated recently, we might be waiting too long between triggers so adjust trigger forward. + // TODO: Be more precise about how to scale when we've experienced recent degenerated GC. Existing code is a very + // rough approximation tuned with very limited workload observations. + avg_cycle_time += degen_cycles * avg_cycle_time; + } + } + double avg_alloc_rate = _allocation_rate.upper_bound(_margin_of_error_sd); log_debug(gc)("%s: average GC time: %.2f ms, allocation rate: %.0f %s/s", _generation->name(), avg_cycle_time * 1000, byte_size_in_proper_unit(avg_alloc_rate), proper_unit_for_byte_size(avg_alloc_rate)); if (avg_cycle_time > allocation_headroom / avg_alloc_rate) { + if (avg_cycle_time > original_cycle_time) { + log_debug(gc)("%s: average GC time adjusted from: %.2f ms to %.2f ms because upward trend in live memory retention", + _generation->name(), original_cycle_time, avg_cycle_time); + } + log_info(gc)("Trigger (%s): Average GC time (%.2f ms) is above the time for average allocation rate (%.0f %sB/s) to deplete free headroom (" SIZE_FORMAT "%s) (margin of error = %.2f)", _generation->name(), avg_cycle_time * 1000, byte_size_in_proper_unit(avg_alloc_rate), proper_unit_for_byte_size(avg_alloc_rate), @@ -278,6 +378,7 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { _generation->name(), avg_cycle_time * 1000, byte_size_in_proper_unit(rate), proper_unit_for_byte_size(rate), byte_size_in_proper_unit(allocation_headroom), proper_unit_for_byte_size(allocation_headroom), + _spike_threshold_sd); _last_trigger = SPIKE; return true; diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp index 962c39c4fba57..dde3144952052 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp @@ -48,6 +48,10 @@ ShenandoahAggressiveHeuristics::ShenandoahAggressiveHeuristics(ShenandoahGenerat void ShenandoahAggressiveHeuristics::choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset, RegionData* data, size_t size, size_t free) { + assert(!ShenandoahHeap::heap()->mode()->is_generational(), "AggressiveHeuristics not appropriate in generational mode"); + + // Note that there's no bound on collection set size. If we try to collect too much memory, we'll get an alloc + // failure during collection and we'll degenerate. for (size_t idx = 0; idx < size; idx++) { ShenandoahHeapRegion* r = data[idx]._region; if (r->garbage() > 0) { diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp index 623281e6e6be8..1dfb70a68ba4b 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp @@ -31,6 +31,8 @@ #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp" +#include "gc/shenandoah/shenandoahOldGeneration.hpp" +#include "gc/shenandoah/shenandoahYoungGeneration.hpp" #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" #include "gc/shenandoah/mode/shenandoahMode.hpp" #include "logging/log.hpp" @@ -56,6 +58,8 @@ ShenandoahHeuristics::ShenandoahHeuristics(ShenandoahGeneration* generation) : _gc_times_learned(0), _gc_time_penalties(0), _gc_time_history(new TruncatedSeq(10, ShenandoahAdaptiveDecayFactor)), + _live_memory_last_cycle(0), + _live_memory_penultimate_cycle(0), _metaspace_oom() { // No unloading during concurrent mark? Communicate that to heuristics @@ -99,9 +103,11 @@ bool ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec size_t free = 0; size_t free_regions = 0; + size_t live_memory = 0; ShenandoahMarkingContext* const ctx = _generation->complete_marking_context(); + size_t remnant_available = 0; for (size_t i = 0; i < num_regions; i++) { ShenandoahHeapRegion* region = heap->get_region(i); if (!in_generation(region)) { @@ -123,13 +129,13 @@ bool ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec } else { assert (_generation->generation_mode() != OLD, "OLD is handled elsewhere"); + live_memory += region->get_live_data_bytes(); // This is our candidate for later consideration. candidates[cand_idx]._region = region; if (heap->mode()->is_generational() && (region->age() >= InitialTenuringThreshold)) { // Bias selection of regions that have reached tenure age for (int i = region->age() - InitialTenuringThreshold; i >= 0; i--) { - // Avoid floating-point math with integer multiply and shift. - garbage = (garbage * ShenandoahTenuredRegionUsageBias) >> ShenandoahTenuredRegionUsageBiasLogBase2; + garbage = (garbage + ShenandoahTenuredRegionUsageBias) * ShenandoahTenuredRegionUsageBias; } } candidates[cand_idx]._garbage = garbage; @@ -151,14 +157,20 @@ bool ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec // Count only the start. Continuations would be counted on "trash" path immediate_regions++; immediate_garbage += garbage; + } else { + live_memory += region->get_live_data_bytes(); } } else if (region->is_trash()) { // Count in just trashed collection set, during coalesced CM-with-UR immediate_regions++; immediate_garbage += garbage; + } else { // region->is_humongous_cont() and !region->is_trash() + live_memory += region->get_live_data_bytes(); } } + save_last_live_memory(live_memory); + // Step 2. Look back at garbage statistics, and decide if we want to collect anything, // given the amount of immediately reclaimable garbage. If we do, figure out the collection set. @@ -175,16 +187,168 @@ bool ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec if (old_heuristics->prime_collection_set(collection_set)) { result = true; } + + size_t bytes_reserved_for_old_evacuation = collection_set->get_old_bytes_reserved_for_evacuation(); + if (bytes_reserved_for_old_evacuation * ShenandoahEvacWaste < heap->get_old_evac_reserve()) { + size_t old_evac_reserve = (size_t) (bytes_reserved_for_old_evacuation * ShenandoahEvacWaste); + heap->set_old_evac_reserve(old_evac_reserve); + } } // else, this is global collection and doesn't need to prime_collection_set + ShenandoahYoungGeneration* young_generation = heap->young_generation(); + size_t young_evacuation_reserve = (young_generation->soft_max_capacity() * ShenandoahEvacReserve) / 100; + + // At this point, young_generation->available() does not know about recently discovered immediate garbage. + // What memory it does think to be available is not entirely trustworthy because any available memory associated + // with a region that is placed into the collection set becomes unavailable when the region is chosen + // for the collection set. We'll compute an approximation of young available. If young_available is zero, + // we'll need to borrow from old-gen in order to evacuate. If there's nothing to borrow, we're going to + // degenerate to full GC. + + // TODO: younng_available can include available (between top() and end()) within each young region that is not + // part of the collection set. Making this memory available to the young_evacuation_reserve allows a larger + // young collection set to be chosen when available memory is under extreme pressure. Implementing this "improvement" + // is tricky, because the incremental construction of the collection set actually changes the amount of memory + // available to hold evacuated young-gen objects. As currently implemented, the memory that is available within + // non-empty regions that are not selected as part of the collection set can be allocated by the mutator while + // GC is evacuating and updating references. + + size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); + size_t free_affiliated_regions = immediate_regions + free_regions; + size_t young_available = (free_affiliated_regions + young_generation->free_unaffiliated_regions()) * region_size_bytes; + + size_t regions_available_to_loan = 0; + size_t preapproved_evac_reserve_loan = 0; + + if (heap->mode()->is_generational()) { + // Now that we've primed the collection set, we can figure out how much memory to reserve for evacuation + // of young-gen objects. + // + // YoungEvacuationReserve for young generation: how much memory are we reserving to hold the results + // of evacuating young collection set regions? This is typically smaller than the total amount + // of available memory, and is also smaller than the total amount of marked live memory within + // young-gen. This value is the minimum of: + // 1. young_gen->available() + (old_gen->available - (OldEvacuationReserve + PromotionReserve)) + // 2. young_gen->capacity() * ShenandoahEvacReserve + // + // Note that any region added to the collection set will be completely evacuated and its memory will + // be completely recycled at the end of GC. The recycled memory will be at least as great as the + // memory borrowed from old-gen. Enforce that the amount borrowed from old-gen for YoungEvacuationReserve + // is an integral number of entire heap regions. + // + young_evacuation_reserve -= heap->get_old_evac_reserve(); + + size_t old_region_borrow_count = 0; + + // Though we cannot know the evacuation_supplement until after we have computed the collection set, we do + // know that every young-gen region added to the collection set will have a net positive impact on available + // memory within young-gen, since each contributes a positive amount of garbage to available. Thus, even + // without knowing the exact composition of the collection set, we can allow young_evacuation_reserve to + // exceed young_available if there are empty regions available within old-gen to hold the results of evacuation. + + ShenandoahGeneration* old_generation = heap->old_generation(); + ShenandoahYoungGeneration* young_generation = heap->young_generation(); + + // Not all of what is currently available within young-gen can be reserved to hold the results of young-gen + // evacuation. This is because memory available within any heap region that is placed into the collection set + // is not available to be allocated during evacuation. To be safe, we assure that all memory required for evacuation + // is available within "virgin" heap regions. + + const size_t available_young_regions = free_regions + immediate_regions + young_generation->free_unaffiliated_regions(); + const size_t available_old_regions = old_generation->free_unaffiliated_regions(); + size_t already_reserved_old_bytes = heap->get_old_evac_reserve() + heap->get_promotion_reserve(); + size_t regions_reserved_for_evac_and_promotion = (already_reserved_old_bytes + region_size_bytes - 1) / region_size_bytes; + regions_available_to_loan = available_old_regions - regions_reserved_for_evac_and_promotion; + + if (available_young_regions * region_size_bytes < young_evacuation_reserve) { + // Try to borrow old-gen regions in order to avoid shrinking young_evacuation_reserve + size_t loan_request = young_evacuation_reserve - available_young_regions * region_size_bytes; + size_t loaned_region_request = (loan_request + region_size_bytes - 1) / region_size_bytes; + if (loaned_region_request > regions_available_to_loan) { + // Scale back young_evacuation_reserve to consume all available young and old regions. After the + // collection set is chosen, we may get some of this memory back for pacing allocations during evacuation + // and update refs. + loaned_region_request = regions_available_to_loan; + young_evacuation_reserve = (available_young_regions + loaned_region_request) * region_size_bytes; + } else { + // No need to scale back young_evacuation_reserve. + } + preapproved_evac_reserve_loan = loaned_region_request * region_size_bytes; + } else { + // No need scale back young_evacuation_reserve and no need to borrow from old-gen. We may even have some + // available_young_regions to support allocation pacing. + } + + } else if (young_evacuation_reserve > young_available) { + // In non-generational mode, there's no old-gen memory to borrow from + young_evacuation_reserve = young_available; + } + + heap->set_young_evac_reserve(young_evacuation_reserve); + heap->reset_young_evac_expended(); + // Add young-gen regions into the collection set. This is a virtual call, implemented differently by each // of the heuristics subclasses. choose_collection_set_from_regiondata(collection_set, candidates, cand_idx, immediate_garbage + free); + size_t young_evacuated_bytes = collection_set->get_young_bytes_reserved_for_evacuation();; + if (young_evacuated_bytes * ShenandoahEvacWaste < young_evacuation_reserve) { + young_evacuation_reserve = (size_t) (young_evacuated_bytes * ShenandoahEvacWaste); + heap->set_young_evac_reserve((size_t) young_evacuation_reserve); + } + + // Now compute the evacuation supplement, which is extra memory borrowed from old-gen that can be allocated + // by mutators while GC is working on evacuation and update-refs. + + // During evacuation and update refs, we will be able to allocate any memory that is currently available + // plus any memory that can be borrowed on the collateral of the current collection set, reserving a certain + // percentage of the anticipated replenishment from collection set memory to be allocated during the subsequent + // concurrent marking effort. This is how much I can repay. + size_t potential_supplement_regions = collection_set->get_young_region_count(); + + // Though I can repay potential_supplement_regions, I can't borrow them unless they are available in old-gen. + if (potential_supplement_regions > regions_available_to_loan) { + potential_supplement_regions = regions_available_to_loan; + } + + size_t potential_evac_supplement; + + // How much of the potential_supplement_regions will be consumed by young_evacuation_reserve: borrowed_evac_regions. + const size_t available_unaffiliated_young_regions = young_generation->free_unaffiliated_regions(); + const size_t available_affiliated_regions = free_regions + immediate_regions; + const size_t available_young_regions = available_unaffiliated_young_regions + available_affiliated_regions; + size_t young_evac_regions = (young_evacuation_reserve + region_size_bytes - 1) / region_size_bytes; + size_t borrowed_evac_regions = (young_evac_regions > available_young_regions)? young_evac_regions - available_young_regions: 0; + + potential_supplement_regions -= borrowed_evac_regions; + potential_evac_supplement = potential_supplement_regions * region_size_bytes; + + // Leave some allocation runway for subsequent concurrent mark phase. + potential_evac_supplement = (potential_evac_supplement * ShenandoahBorrowPercent) / 100; + + heap->set_alloc_supplement_reserve(potential_evac_supplement); + + size_t promotion_budget = heap->get_promotion_reserve(); + size_t old_evac_budget = heap->get_old_evac_reserve(); + size_t alloc_budget_evac_and_update = potential_evac_supplement + young_available; + + // TODO: young_available, which feeds into alloc_budget_evac_and_update is lacking memory available within + // existing young-gen regions that were not selected for the collection set. Add this in and adjust the + // log message (where it says "empty-region allocation budget"). + + log_info(gc, ergo)("Memory reserved for evacuation and update-refs includes promotion budget: " SIZE_FORMAT + "%s, young evacuation budget: " SIZE_FORMAT "%s, old evacuation budget: " SIZE_FORMAT + "%s, empty-region allocation budget: " SIZE_FORMAT "%s, including supplement: " SIZE_FORMAT "%s", + byte_size_in_proper_unit(promotion_budget), proper_unit_for_byte_size(promotion_budget), + byte_size_in_proper_unit(young_evacuation_reserve), proper_unit_for_byte_size(young_evacuation_reserve), + byte_size_in_proper_unit(old_evac_budget), proper_unit_for_byte_size(old_evac_budget), + byte_size_in_proper_unit(alloc_budget_evac_and_update), + proper_unit_for_byte_size(alloc_budget_evac_and_update), + byte_size_in_proper_unit(potential_evac_supplement), proper_unit_for_byte_size(potential_evac_supplement)); } + // else, we're going to skip evacuation and update refs because we reclaimed sufficient amounts of immediate garbage. size_t cset_percent = (total_garbage == 0) ? 0 : (collection_set->garbage() * 100 / total_garbage); - size_t collectable_garbage = collection_set->garbage() + immediate_garbage; size_t collectable_garbage_percent = (total_garbage == 0) ? 0 : (collectable_garbage * 100 / total_garbage); @@ -327,3 +491,15 @@ bool ShenandoahHeuristics::in_generation(ShenandoahHeapRegion* region) { || (_generation->generation_mode() == OLD && region->affiliation() == OLD_GENERATION)); } +void ShenandoahHeuristics::save_last_live_memory(size_t live_memory) { + _live_memory_penultimate_cycle = _live_memory_last_cycle; + _live_memory_last_cycle = live_memory; +} + +size_t ShenandoahHeuristics::get_last_live_memory() { + return _live_memory_last_cycle; +} + +size_t ShenandoahHeuristics::get_penultimate_live_memory() { + return _live_memory_penultimate_cycle; +} diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp index fe6f9da9b5df5..aa96c81d01a7a 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp @@ -99,6 +99,9 @@ class ShenandoahHeuristics : public CHeapObj { intx _gc_time_penalties; TruncatedSeq* _gc_time_history; + size_t _live_memory_last_cycle; + size_t _live_memory_penultimate_cycle; + // There may be many threads that contend to set this flag ShenandoahSharedFlag _metaspace_oom; @@ -128,6 +131,10 @@ class ShenandoahHeuristics : public CHeapObj { _guaranteed_gc_interval = guaranteed_gc_interval; } + uint degenerated_cycles_in_a_row() { + return _degenerated_cycles_in_a_row; + } + virtual void record_cycle_start(); virtual void record_cycle_end(); @@ -159,6 +166,10 @@ class ShenandoahHeuristics : public CHeapObj { virtual void initialize(); double time_since_last_gc() const; + + void save_last_live_memory(size_t live_memory); + size_t get_last_live_memory(); + size_t get_penultimate_live_memory(); }; #endif // SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHHEURISTICS_HPP diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp index 8d8f9e13faa14..395c255bb78ad 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp @@ -25,6 +25,9 @@ #include "precompiled.hpp" #include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp" +#include "gc/shenandoah/shenandoahGeneration.hpp" +#include "gc/shenandoah/shenandoahOldGeneration.hpp" +#include "gc/shenandoah/shenandoahYoungGeneration.hpp" #include "utilities/quickSort.hpp" ShenandoahOldHeuristics::ShenandoahOldHeuristics(ShenandoahGeneration* generation, ShenandoahHeuristics* trigger_heuristic) : @@ -42,10 +45,10 @@ ShenandoahOldHeuristics::ShenandoahOldHeuristics(ShenandoahGeneration* generatio bool ShenandoahOldHeuristics::prime_collection_set(ShenandoahCollectionSet* collection_set) { if (unprocessed_old_collection_candidates() == 0) { - // no candidates for inclusion in collection set. return false; } + ShenandoahHeap* heap = ShenandoahHeap::heap(); uint included_old_regions = 0; size_t evacuated_old_bytes = 0; @@ -55,25 +58,38 @@ bool ShenandoahOldHeuristics::prime_collection_set(ShenandoahCollectionSet* coll // by heuristics and should adjust dynamically based on most current execution behavior. In the // interim, we offer command-line options to set the values of these configuration parameters. - // max_old_evacuation_bytes represents an "arbitrary" bound on how much evacuation effort is dedicated + // max_old_evacuation_bytes represents a bound on how much evacuation effort is dedicated // to old-gen regions. - const size_t max_old_evacuation_bytes = (size_t) ((double)_generation->soft_max_capacity() / 100 * ShenandoahOldEvacReserve); + size_t max_old_evacuation_bytes = (heap->old_generation()->soft_max_capacity() * ShenandoahOldEvacReserve) / 100; + const size_t young_evacuation_bytes = (heap->young_generation()->soft_max_capacity() * ShenandoahEvacReserve) / 100; + const size_t ratio_bound_on_old_evac_bytes = (young_evacuation_bytes * ShenandoahOldEvacRatioPercent) / 100; + if (max_old_evacuation_bytes > ratio_bound_on_old_evac_bytes) { + max_old_evacuation_bytes = ratio_bound_on_old_evac_bytes; + } + + // Usually, old-evacuation is limited by the CPU bounds on effort. However, it can also be bounded by available + // memory within old-gen to hold the results of evacuation. When we are bound by memory availability, we need + // to account below for the loss of available memory from within each region that is added to the old-gen collection + // set. + size_t old_available = heap->old_generation()->available(); + size_t excess_old_capacity_for_evacuation; + if (max_old_evacuation_bytes > old_available) { + max_old_evacuation_bytes = old_available; + excess_old_capacity_for_evacuation = 0; + } else { + excess_old_capacity_for_evacuation = old_available - max_old_evacuation_bytes; + } // promotion_budget_bytes represents an "arbitrary" bound on how many bytes can be consumed by young-gen // objects promoted into old-gen memory. We need to avoid a scenario under which promotion of objects // depletes old-gen available memory to the point that there is insufficient memory to hold old-gen objects // that need to be evacuated from within the old-gen collection set. // - // TODO We should probably enforce this, but there is no enforcement currently. Key idea: if there is not - // sufficient memory within old-gen to hold an object that wants to be promoted, defer promotion until a - // subsequent evacuation pass. Since enforcement may be expensive, requiring frequent synchronization - // between mutator and GC threads, here's an alternative "greedy" mitigation strategy: Set the parameter's - // value so overflow is "very rare". In the case that we experience overflow, evacuate what we can from - // within the old collection set, but don't evacuate everything. At the end of evacuation, any collection - // set region that was not fully evacuated cannot be recycled. It becomes a prime candidate for the next - // collection set selection. Here, we'd rather fall back to this contingent behavior than force a full STW - // collection. - const size_t promotion_budget_bytes = (ShenandoahHeapRegion::region_size_bytes() / 2); + // Key idea: if there is not sufficient memory within old-gen to hold an object that wants to be promoted, defer + // promotion until a subsequent evacuation pass. Enforcement is provided at the time PLABs and shared allocations + // in old-gen memory are requested. + + const size_t promotion_budget_bytes = heap->get_promotion_reserve(); // old_evacuation_budget is an upper bound on the amount of live memory that can be evacuated. // @@ -83,29 +99,13 @@ bool ShenandoahOldHeuristics::prime_collection_set(ShenandoahCollectionSet* coll // of live memory in that region and by the amount of unallocated memory in that region if the evacuation // budget is constrained by availability of free memory. See remaining_old_evacuation_budget below. - // Allow no more evacuation than exists free-space within old-gen memory - size_t old_evacuation_budget = ((_generation->available() > promotion_budget_bytes) - ? _generation->available() - promotion_budget_bytes: 0); - - // But if the amount of available free space in old-gen memory exceeds the pacing bound on how much old-gen - // memory can be evacuated during each evacuation pass, then cut the old-gen evacuation further. The pacing - // bound is designed to assure that old-gen evacuations to not excessively slow the evacuation pass in order - // to assure that young-gen GC cadence is not disrupted. - - // excess_free_capacity represents availability of memory to hold evacuations beyond what is required to hold - // planned evacuations. It may go negative if we choose to collect regions with large amounts of free memory. - long long excess_free_capacity; - if (old_evacuation_budget > max_old_evacuation_bytes) { - excess_free_capacity = old_evacuation_budget - max_old_evacuation_bytes; - old_evacuation_budget = max_old_evacuation_bytes; - } else - excess_free_capacity = 0; - - log_info(gc)("Choose old regions for mixed collection: excess capacity: " SIZE_FORMAT "%s, evacuation budget: " SIZE_FORMAT "%s", - byte_size_in_proper_unit((size_t) excess_free_capacity), proper_unit_for_byte_size(excess_free_capacity), + size_t old_evacuation_budget = (size_t) (max_old_evacuation_bytes / ShenandoahEvacWaste); + + log_info(gc)("Choose old regions for mixed collection: old evacuation budget: " SIZE_FORMAT "%s", byte_size_in_proper_unit(old_evacuation_budget), proper_unit_for_byte_size(old_evacuation_budget)); size_t remaining_old_evacuation_budget = old_evacuation_budget; + size_t lost_evacuation_capacity = 0; // The number of old-gen regions that were selected as candidates for collection at the end of the most recent // old-gen concurrent marking phase and have not yet been collected is represented by @@ -114,36 +114,35 @@ bool ShenandoahOldHeuristics::prime_collection_set(ShenandoahCollectionSet* coll // Old collection candidates are sorted in order of decreasing garbage contained therein. ShenandoahHeapRegion* r = next_old_collection_candidate(); - // Assuming region r is added to the collection set, what will be the remaining_old_evacuation_budget after - // accounting for the loss of region r's free() memory. - size_t adjusted_remaining_old_evacuation_budget; // If we choose region r to be collected, then we need to decrease the capacity to hold other evacuations by // the size of r's free memory. - excess_free_capacity -= r->free(); - // If subtracting r->free from excess_free_capacity() makes it go negative, that means we are going to have - // to decrease the evacuation budget. - if (excess_free_capacity < 0) { - if (remaining_old_evacuation_budget < (size_t) -excess_free_capacity) { - // By setting adjusted_remaining_old_evacuation_budget to 0, we prevent further additions to the old-gen - // collection set, unless the region has zero live data bytes. - adjusted_remaining_old_evacuation_budget = 0; - } else { - // Adding negative excess_free_capacity decreases the adjusted_remaining_old_evacuation_budget - adjusted_remaining_old_evacuation_budget = remaining_old_evacuation_budget + excess_free_capacity; + if ((r->get_live_data_bytes() <= remaining_old_evacuation_budget) && + ((lost_evacuation_capacity + r->free() <= excess_old_capacity_for_evacuation) + || (r->get_live_data_bytes() + r->free() <= remaining_old_evacuation_budget))) { + + // Decrement remaining evacuation budget by bytes that will be copied. If the cumulative loss of free memory from + // regions that are to be collected exceeds excess_old_capacity_for_evacuation, decrease + // remaining_old_evacuation_budget by this loss as well. + lost_evacuation_capacity += r->free(); + remaining_old_evacuation_budget -= r->get_live_data_bytes(); + if (lost_evacuation_capacity > excess_old_capacity_for_evacuation) { + // This is slightly conservative because we really only need to remove from the remaining evacuation budget + // the amount by which lost_evacution_capacity exceeds excess_old_capacity_for_evacuation, but this is relatively + // rare event and current thought is to be a bit conservative rather than mess up the math on code that is so + // difficult to test and maintain... + + // Once we have crossed the threshold of lost_evacuation_capacity exceeding excess_old_capacity_for_evacuation, + // every subsequent iteration of this loop will further decrease remaining_old_evacuation_budget. + remaining_old_evacuation_budget -= r->free(); } + collection_set->add_region(r); + included_old_regions++; + evacuated_old_bytes += r->get_live_data_bytes(); + consume_old_collection_candidate(); } else { - adjusted_remaining_old_evacuation_budget = remaining_old_evacuation_budget; - } - - if (r->get_live_data_bytes() > adjusted_remaining_old_evacuation_budget) { break; } - collection_set->add_region(r); - included_old_regions++; - evacuated_old_bytes += r->get_live_data_bytes(); - consume_old_collection_candidate(); - remaining_old_evacuation_budget = adjusted_remaining_old_evacuation_budget - r->get_live_data_bytes(); } if (included_old_regions > 0) { @@ -154,7 +153,10 @@ bool ShenandoahOldHeuristics::prime_collection_set(ShenandoahCollectionSet* coll } // Both arguments are don't cares for old-gen collections -bool ShenandoahOldHeuristics::choose_collection_set(ShenandoahCollectionSet* collection_set, ShenandoahOldHeuristics* old_heuristics) { +bool ShenandoahOldHeuristics::choose_collection_set(ShenandoahCollectionSet* collection_set, + ShenandoahOldHeuristics* old_heuristics) { + assert((collection_set == nullptr) && (old_heuristics == nullptr), + "Expect null arguments in ShenandoahOldHeuristics::choose_collection_set()"); // Old-gen doesn't actually choose a collection set to be evacuated by its own gang of worker tasks. // Instead, it computes the set of regions to be evacuated by subsequent young-gen evacuation passes. prepare_for_old_collections(); diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp index 2e7da1f1dd257..f4df45133a452 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp @@ -54,7 +54,7 @@ void ShenandoahPassiveHeuristics::choose_collection_set_from_regiondata(Shenando // Do not select too large CSet that would overflow the available free space. // Take at least the entire evacuation reserve, and be free to overflow to free space. size_t max_capacity = ShenandoahHeap::heap()->max_capacity(); - size_t available = MAX2(max_capacity / 100 * ShenandoahEvacReserve, actual_free); + size_t available = MAX2((max_capacity / 100) * ShenandoahEvacReserve, actual_free); size_t max_cset = (size_t)(available / ShenandoahEvacWaste); log_info(gc, ergo)("CSet Selection. Actual Free: " SIZE_FORMAT "%s, Max CSet: " SIZE_FORMAT "%s", diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp index 2d7d4daee6a4d..773b38ebe476d 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp @@ -58,6 +58,19 @@ const char* affiliation_name(oop ptr) { return affiliation_name(region->affiliation()); } +const char affiliation_code(ShenandoahRegionAffiliation type) { + switch(type) { + case ShenandoahRegionAffiliation::FREE: + return 'F'; + case ShenandoahRegionAffiliation::YOUNG_GENERATION: + return 'Y'; + case ShenandoahRegionAffiliation::OLD_GENERATION: + return 'O'; + default: + ShouldNotReachHere(); + return 'X'; + } +} const char* affiliation_name(ShenandoahRegionAffiliation type) { switch (type) { diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp index 655e2b9fac5a2..5a5fefd98e804 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp @@ -42,6 +42,7 @@ enum ShenandoahRegionAffiliation { const char* affiliation_name(oop ptr); const char* affiliation_name(ShenandoahRegionAffiliation type); +const char affiliation_code(ShenandoahRegionAffiliation type); class ShenandoahGenerationalMode : public ShenandoahMode { public: diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp index fe5250944081d..2cabf498d8a3d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp @@ -87,6 +87,15 @@ void ShenandoahCollectionSet::add_region(ShenandoahHeapRegion* r) { assert(!r->is_humongous(), "Only add regular regions to the collection set"); _cset_map[r->index()] = 1; + + if (r->affiliation() == YOUNG_GENERATION) { + _young_region_count++; + _young_bytes_to_evacuate += r->get_live_data_bytes(); + } else if (r->affiliation() == OLD_GENERATION) { + _old_region_count++; + _old_bytes_to_evacuate += r->get_live_data_bytes(); + } + _region_count++; _has_old_regions |= r->is_old(); _garbage += r->garbage(); @@ -111,6 +120,11 @@ void ShenandoahCollectionSet::clear() { _region_count = 0; _current_index = 0; + _young_region_count = 0; + _old_region_count = 0; + _young_bytes_to_evacuate = 0; + _old_bytes_to_evacuate = 0; + _has_old_regions = false; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp index 34f356d13df87..6c3398c5499c1 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp @@ -47,6 +47,15 @@ class ShenandoahCollectionSet : public CHeapObj { size_t _garbage; size_t _used; size_t _region_count; + size_t _immediate_trash; + size_t _evacuation_reserve; // How many bytes reserved in generation for evacuation replicas. This does + // not include bytes reserved for old-generation replicas. The value is + // conservative in that memory may be reserved for objects that will be promoted. + size_t _young_bytes_to_evacuate; + size_t _old_bytes_to_evacuate; + + size_t _young_region_count; + size_t _old_region_count; shenandoah_padding(0); volatile size_t _current_index; @@ -78,6 +87,25 @@ class ShenandoahCollectionSet : public CHeapObj { void print_on(outputStream* out) const; + inline size_t get_immediate_trash(); + inline void set_immediate_trash(size_t immediate_trash); + + // This represents total amount of work to be performed by evacuation, including evacuations to young, to old, + // and promotions from young to old. This equals get_young_bytes_reserved_for_evacuation() plus + // get_old_bytes_reserved_for_evacuation(). + inline size_t get_bytes_reserved_for_evacuation(); + + // It is not known how many of these bytes will be promoted. + inline size_t get_young_bytes_reserved_for_evacuation(); + inline void reserve_young_bytes_for_evacuation(size_t byte_count); + + inline size_t get_old_bytes_reserved_for_evacuation(); + inline void reserve_old_bytes_for_evacuation(size_t byte_count); + + inline size_t get_old_region_count(); + + inline size_t get_young_region_count(); + bool has_old_regions() const { return _has_old_regions; } size_t used() const { return _used; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.inline.hpp index ba73e5e638202..21bda091ea306 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.inline.hpp @@ -53,4 +53,32 @@ bool ShenandoahCollectionSet::is_in_loc(void* p) const { return _biased_cset_map[index] == 1; } +void ShenandoahCollectionSet::set_immediate_trash(size_t immediate_trash) { + _immediate_trash = immediate_trash; +} + +size_t ShenandoahCollectionSet::get_immediate_trash() { + return _immediate_trash; +} + +size_t ShenandoahCollectionSet::get_old_bytes_reserved_for_evacuation() { + return _old_bytes_to_evacuate; +} + +size_t ShenandoahCollectionSet::get_young_bytes_reserved_for_evacuation() { + return _young_bytes_to_evacuate; +} + +size_t ShenandoahCollectionSet::get_bytes_reserved_for_evacuation() { + return _young_bytes_to_evacuate + _old_bytes_to_evacuate; +} + +size_t ShenandoahCollectionSet::get_old_region_count() { + return _old_region_count; +} + +size_t ShenandoahCollectionSet::get_young_region_count() { + return _young_region_count; +} + #endif // SHARE_GC_SHENANDOAH_SHENANDOAHCOLLECTIONSET_INLINE_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.cpp index 57132ac24b003..adb40e9d2af12 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.cpp @@ -75,6 +75,7 @@ void ShenandoahCollectorPolicy::record_alloc_failure_to_degenerated(ShenandoahGC } void ShenandoahCollectorPolicy::record_degenerated_upgrade_to_full() { + ShenandoahHeap::heap()->record_upgrade_to_full(); _alloc_failure_degenerated_upgrade_to_full++; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index d989970e96ece..f5a9c2184b1ee 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -31,6 +31,8 @@ #include "gc/shenandoah/shenandoahConcurrentGC.hpp" #include "gc/shenandoah/shenandoahFreeSet.hpp" #include "gc/shenandoah/shenandoahGeneration.hpp" +#include "gc/shenandoah/shenandoahOldGeneration.hpp" +#include "gc/shenandoah/shenandoahYoungGeneration.hpp" #include "gc/shenandoah/shenandoahLock.hpp" #include "gc/shenandoah/shenandoahMark.inline.hpp" #include "gc/shenandoah/shenandoahMonitoringSupport.hpp" @@ -86,6 +88,7 @@ ShenandoahGC::ShenandoahDegenPoint ShenandoahConcurrentGC::degen_point() const { bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) { ShenandoahHeap* const heap = ShenandoahHeap::heap(); + heap->start_conc_gc(); if (cause == GCCause::_wb_breakpoint) { ShenandoahBreakpoint::start_gc(); } @@ -198,8 +201,38 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) { // Update references freed up collection set, kick the cleanup to reclaim the space. entry_cleanup_complete(); } else { - vmop_entry_final_roots(); + // We chose not to evacuate because we found sufficient immediate garbage. + vmop_entry_final_roots(heap->is_aging_cycle()); } + size_t old_available, young_available; + { + ShenandoahYoungGeneration* young_gen = heap->young_generation(); + ShenandoahGeneration* old_gen = heap->old_generation(); + ShenandoahHeapLocker locker(heap->lock()); + + size_t old_usage_before_evac = heap->capture_old_usage(0); + size_t old_usage_now = old_gen->used(); + size_t promoted_bytes = old_usage_now - old_usage_before_evac; + heap->set_previous_promotion(promoted_bytes); + + young_gen->unadjust_available(); + old_gen->unadjust_available(); + young_gen->increase_used(heap->get_young_evac_expended()); + // No need to old_gen->increase_used(). That was done when plabs were allocated, accounting for both old evacs and promotions. + + young_available = young_gen->adjusted_available(); + old_available = old_gen->adjusted_available(); + + heap->set_alloc_supplement_reserve(0); + heap->set_young_evac_reserve(0); + heap->reset_young_evac_expended(); + heap->set_old_evac_reserve(0); + heap->reset_old_evac_expended(); + heap->set_promotion_reserve(0); + } + log_info(gc, ergo)("At end of concurrent GC, old_available: " SIZE_FORMAT "%s, young_available: " SIZE_FORMAT "%s", + byte_size_in_proper_unit(old_available), proper_unit_for_byte_size(old_available), + byte_size_in_proper_unit(young_available), proper_unit_for_byte_size(young_available)); return true; } @@ -244,14 +277,14 @@ void ShenandoahConcurrentGC::vmop_entry_final_updaterefs() { VMThread::execute(&op); } -void ShenandoahConcurrentGC::vmop_entry_final_roots() { +void ShenandoahConcurrentGC::vmop_entry_final_roots(bool increment_region_ages) { ShenandoahHeap* const heap = ShenandoahHeap::heap(); TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters()); ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_roots_gross); // This phase does not use workers, no need for setup heap->try_inject_alloc_failure(); - VM_ShenandoahFinalRoots op(this); + VM_ShenandoahFinalRoots op(this, increment_region_ages); VMThread::execute(&op); } @@ -631,7 +664,36 @@ void ShenandoahConcurrentGC::op_final_mark() { // Notify JVMTI that the tagmap table will need cleaning. JvmtiTagMap::set_needs_cleaning(); + // The collection set is chosen by prepare_regions_and_collection_set(). + // + // TODO: Under severe memory overload conditions that can be checked here, we may want to limit + // the inclusion of old-gen candidates within the collection set. This would allow us to prioritize efforts on + // evacuating young-gen, This remediation is most appropriate when old-gen availability is very high (so there + // are negligible negative impacts from delaying completion of old-gen evacuation) and when young-gen collections + // are "under duress" (as signalled by very low availability of memory within young-gen, indicating that/ young-gen + // collections are not triggering frequently enough). bool mixed_evac = _generation->prepare_regions_and_collection_set(true /*concurrent*/); + + // Upon return from prepare_regions_and_collection_set(), certain parameters have been established to govern the + // evacuation efforts that are about to begin. In particular: + // + // heap->get_promotion_reserve() represents the amount of memory within old-gen's available memory that has + // been set aside to hold objects promoted from young-gen memory. This represents an estimated percentage + // of the live young-gen memory within the collection set. If there is more data ready to be promoted than + // can fit within this reserve, the promotion of some objects will be deferred until a subsequent evacuation + // pass. + // + // heap->get_old_evac_reserve() represents the amount of memory within old-gen's available memory that has been + // set aside to hold objects evacuated from the old-gen collection set. + // + // heap->get_young_evac_reserve() represents the amount of memory within young-gen's available memory that has + // been set aside to hold objects evacuated from the young-gen collection set. Conservatively, this value + // equals the entire amount of live young-gen memory within the collection set, even though some of this memory + // will likely be promoted. + // + // heap->get_alloc_supplement_reserve() represents the amount of old-gen memory that can be allocated during evacuation + // and update-refs phases of gc. The young evacuation reserve has already been removed from this quantity. + heap->set_mixed_evac(mixed_evac); // Has to be done after cset selection @@ -659,6 +721,20 @@ void ShenandoahConcurrentGC::op_final_mark() { // Notify JVMTI that oops are changed. JvmtiTagMap::set_needs_rehashing(); + if (heap->mode()->is_generational()) { + // Calculate the temporary evacuation allowance supplement to young-gen memory capacity (for allocations + // and young-gen evacuations). + size_t young_available = heap->young_generation()->adjust_available(heap->get_alloc_supplement_reserve()); + // old_available is memory that can hold promotions and evacuations. Subtract out the memory that is being + // loaned for young-gen allocations or evacuations. + size_t old_available = heap->old_generation()->adjust_available(-heap->get_alloc_supplement_reserve()); + + log_info(gc, ergo)("After generational memory budget adjustments, old avaiable: " SIZE_FORMAT + "%s, young_available: " SIZE_FORMAT "%s", + byte_size_in_proper_unit(old_available), proper_unit_for_byte_size(old_available), + byte_size_in_proper_unit(young_available), proper_unit_for_byte_size(young_available)); + } + if (ShenandoahPacing) { heap->pacer()->setup_for_evac(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp index 57957a5378bcd..c82408b7767dc 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp @@ -67,7 +67,7 @@ class ShenandoahConcurrentGC : public ShenandoahGC { protected: void vmop_entry_final_mark(); - void vmop_entry_final_roots(); + void vmop_entry_final_roots(bool incr_region_ages); private: void vmop_entry_init_updaterefs(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp index c90ebd21a70ae..b633ea5dcc1f1 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp @@ -38,6 +38,7 @@ #include "gc/shenandoah/shenandoahSTWMark.hpp" #include "gc/shenandoah/shenandoahUtils.hpp" #include "gc/shenandoah/shenandoahVerifier.hpp" +#include "gc/shenandoah/shenandoahYoungGeneration.hpp" #include "gc/shenandoah/shenandoahWorkerPolicy.hpp" #include "gc/shenandoah/shenandoahVMOperations.hpp" #include "runtime/vmThread.hpp" @@ -225,6 +226,24 @@ void ShenandoahDegenGC::op_degenerated() { ShouldNotReachHere(); } + if (heap->mode()->is_generational()) { + // In case degeneration interrupted concurrent evacuation or update references, we need to clean up transient state. + // Otherwise, these actions have no effect. + + heap->young_generation()->unadjust_available(); + heap->old_generation()->unadjust_available(); + heap->young_generation()->increase_used(heap->get_young_evac_expended()); + // No need to old_gen->increase_used(). That was done when plabs were allocated, accounting for both old evacs and promotions. + + heap->set_alloc_supplement_reserve(0); + heap->set_young_evac_reserve(0); + heap->reset_young_evac_expended(); + heap->set_old_evac_reserve(0); + heap->reset_old_evac_expended(); + heap->set_promotion_reserve(0); + + } + if (ShenandoahVerify) { heap->verifier()->verify_after_degenerated(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index fa92c1ffce123..881f784418394 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -94,13 +94,15 @@ HeapWord* ShenandoahFreeSet::allocate_single(ShenandoahAllocRequest& req, bool& // Free set maintains mutator and collector views, and normally they allocate in their views only, // unless we special cases for stealing and mixed allocations. + // Overwrite with non-zero (non-NULL) values only if necessary for allocation bookkeeping. + switch (req.type()) { case ShenandoahAllocRequest::_alloc_tlab: case ShenandoahAllocRequest::_alloc_shared: { - // Try to allocate in the mutator view for (size_t idx = _mutator_leftmost; idx <= _mutator_rightmost; idx++) { if (is_mutator_free(idx)) { + // try_allocate_in() increases used if the allocation is successful. HeapWord* result = try_allocate_in(_heap->get_region(idx), req, in_new_region); if (result != NULL) { return result; @@ -112,7 +114,12 @@ HeapWord* ShenandoahFreeSet::allocate_single(ShenandoahAllocRequest& req, bool& break; } case ShenandoahAllocRequest::_alloc_gclab: + // GCLABs are for evacuation so we must be in evacuation phase. If this allocation is successful, increment + // the relevant evac_expended rather than used value. + case ShenandoahAllocRequest::_alloc_plab: + // PLABs always reside in old-gen and are only allocated during evacuation phase. + case ShenandoahAllocRequest::_alloc_shared_gc: { // First try to fit into a region that is already in use in the same generation. HeapWord* result = allocate_with_affiliation(req.affiliation(), req, in_new_region); @@ -198,12 +205,16 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah HeapWord* result = NULL; size_t size = req.size(); - // req.size() is in words, free() is in bytes. + // req.size() is in words, r->free() is in bytes. if (ShenandoahElasticTLAB && req.is_lab_alloc()) { if (req.type() == ShenandoahAllocRequest::_alloc_plab) { // Need to assure that plabs are aligned on multiple of card region. size_t free = r->free(); size_t usable_free = (free / CardTable::card_size) << CardTable::card_shift; + if ((free != usable_free) && (free - usable_free < ShenandoahHeap::min_fill_size() * HeapWordSize)) { + // We'll have to add another card's memory to the padding + usable_free -= CardTable::card_size; + } free /= HeapWordSize; usable_free /= HeapWordSize; if (size > usable_free) { @@ -222,6 +233,7 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah } } } else { + // This is a GCLAB or a TLAB allocation size_t free = align_down(r->free() >> LogHeapWordSize, MinObjAlignment); if (size > free) { size = free; @@ -236,6 +248,10 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah size_t usable_free = (free / CardTable::card_size) << CardTable::card_shift; free /= HeapWordSize; usable_free /= HeapWordSize; + if ((free != usable_free) && (free - usable_free < ShenandoahHeap::min_fill_size() * HeapWordSize)) { + // We'll have to add another card's memory to the padding + usable_free -= CardTable::card_size; + } if (size <= usable_free) { assert(size % CardTable::card_size_in_words == 0, "PLAB size must be multiple of remembered set card size"); @@ -260,8 +276,12 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah // Allocation successful, bump stats: if (req.is_mutator_alloc()) { + // Mutator allocations always pull from young gen. + _heap->young_generation()->increase_used(size * HeapWordSize); increase_used(size * HeapWordSize); - } else if (req.is_gc_alloc()) { + } else { + // assert(req.is_gc_alloc(), "Should be gc_alloc since req wasn't mutator alloc"); + // For GC allocations, we advance update_watermark because the objects relocated into this memory during // evacuation are not updated during evacuation. For both young and old regions r, it is essential that all // PLABs be made parsable at the end of evacuation. This is enabled by retiring all plabs at end of evacuation. @@ -270,22 +290,29 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah // PLABs parsable while still allowing the PLAB to serve future allocation requests that arise during the // next evacuation pass. r->set_update_watermark(r->top()); - } - if (r->affiliation() == ShenandoahRegionAffiliation::YOUNG_GENERATION) { - _heap->young_generation()->increase_used(size * HeapWordSize); - } else if (r->affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) { - _heap->old_generation()->increase_used(size * HeapWordSize); + if (r->affiliation() == ShenandoahRegionAffiliation::YOUNG_GENERATION) { + // This is either a GCLAB or it is a shared evacuation allocation. In either case, we expend young evac. + // At end of update refs, we'll add expended young evac into young_gen->used. We hide this usage + // from current accounting because memory reserved for evacuation is not part of adjusted capacity. + _heap->expend_young_evac(size * HeapWordSize); + } else { + assert(r->affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION, "GC Alloc was not YOUNG so must be OLD"); + + assert(req.type() != _alloc_gclab, "old-gen allocations use PLAB or shared allocation"); + _heap->old_generation()->increase_used(size * HeapWordSize); + // for plabs, we'll sort the difference between evac and promotion usage when we retire the plab + } } } - if (result == NULL || has_no_alloc_capacity(r)) { // Region cannot afford this or future allocations. Retire it. // // While this seems a bit harsh, especially in the case when this large allocation does not // fit, but the next small one would, we are risking to inflate scan times when lots of - // almost-full regions precede the fully-empty region where we want allocate the entire TLAB. - // TODO: Record first fully-empty region, and use that for large allocations + // almost-full regions precede the fully-empty region where we want to allocate the entire TLAB. + // TODO: Record first fully-empty region, and use that for large allocations and/or organize + // available free segments within regions for more efficient searches for "good fit". // Record the remainder as allocation waste if (req.is_mutator_alloc()) { @@ -426,7 +453,6 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req) { // While individual regions report their true use, all humongous regions are // marked used in the free set. increase_used(ShenandoahHeapRegion::region_size_bytes() * num); - if (req.affiliation() == ShenandoahRegionAffiliation::YOUNG_GENERATION) { _heap->young_generation()->increase_used(words_size * HeapWordSize); } else if (req.affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) { @@ -551,7 +577,7 @@ void ShenandoahFreeSet::rebuild() { } // Evac reserve: reserve trailing space for evacuations - size_t to_reserve = _heap->max_capacity() / 100 * ShenandoahEvacReserve; + size_t to_reserve = (_heap->max_capacity() / 100) * ShenandoahEvacReserve; size_t reserved = 0; for (size_t idx = _heap->num_regions() - 1; idx > 0; idx--) { @@ -667,6 +693,7 @@ HeapWord* ShenandoahFreeSet::allocate(ShenandoahAllocRequest& req, bool& in_new_ shenandoah_assert_heaplocked(); assert_bounds(); + // Allocation request is known to satisfy all memory budgeting constraints. if (req.size() > ShenandoahHeapRegion::humongous_threshold_words()) { switch (req.type()) { case ShenandoahAllocRequest::_alloc_shared: diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp index e59df83b28c4b..98a9a66f22e44 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp @@ -69,9 +69,6 @@ class ShenandoahFreeSet : public CHeapObj { void increase_used(size_t amount); void clear_internal(); - size_t collector_count() const { return _collector_free_bitmap.count_one_bits(); } - size_t mutator_count() const { return _mutator_free_bitmap.count_one_bits(); } - void try_recycle_trashed(ShenandoahHeapRegion *r); bool can_allocate_from(ShenandoahHeapRegion *r); @@ -81,6 +78,12 @@ class ShenandoahFreeSet : public CHeapObj { public: ShenandoahFreeSet(ShenandoahHeap* heap, size_t max_regions); + // Number of regions dedicated to GC allocations (for evacuation or promotion) that are currently free + size_t collector_count() const { return _collector_free_bitmap.count_one_bits(); } + + // Number of regions dedicated to mutator allocations that are currently free + size_t mutator_count() const { return _mutator_free_bitmap.count_one_bits(); } + void clear(); void rebuild(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index 345fa691efad0..e2fb095b9e1b3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -181,6 +181,19 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) { // Since we may arrive here from degenerated GC failure of either young or old, establish generation as GLOBAL. heap->set_gc_generation(heap->global_generation()); + // There will be no concurrent allocations during full GC so reset these coordination variables. + heap->young_generation()->unadjust_available(); + heap->old_generation()->unadjust_available(); + heap->young_generation()->increase_used(heap->get_young_evac_expended()); + // No need to old_gen->increase_used(). That was done when plabs were allocated, accounting for both old evacs and promotions. + + heap->set_alloc_supplement_reserve(0); + heap->set_young_evac_reserve(0); + heap->reset_young_evac_expended(); + heap->set_old_evac_reserve(0); + heap->reset_old_evac_expended(); + heap->set_promotion_reserve(0); + if (ShenandoahVerify) { heap->verifier()->verify_before_fullgc(); } @@ -402,6 +415,10 @@ class ShenandoahPrepareForGenerationalCompactionObjectClosure : public ObjectClo ShenandoahPrepareForCompactionTask* _compactor; PreservedMarks* const _preserved_marks; ShenandoahHeap* const _heap; + + // _empty_regions is a thread-local list of heap regions that have been completely emptied by this worker thread's + // compaction efforts. The worker thread that drives these efforts adds compacted regions to this list if the + // region has not been compacted onto itself. GrowableArray& _empty_regions; int _empty_regions_pos; ShenandoahHeapRegion* _old_to_region; @@ -488,8 +505,34 @@ class ShenandoahPrepareForGenerationalCompactionObjectClosure : public ObjectClo assert(!_heap->complete_marking_context()->allocated_after_mark_start(p), "must be truly marked"); size_t obj_size = p->size(); - if (_from_affiliation == ShenandoahRegionAffiliation::OLD_GENERATION) { - assert(_old_to_region != nullptr, "_old_to_region should not be NULL when compacting OLD _from_region"); + uint from_region_age = _from_region->age(); + uint object_age = p->age(); + + bool promote_object = false; + if ((_from_affiliation == ShenandoahRegionAffiliation::YOUNG_GENERATION) && + (from_region_age + object_age > InitialTenuringThreshold)) { + if ((_old_to_region != nullptr) && (_old_compact_point + obj_size > _old_to_region->end())) { + finish_old_region(); + _old_to_region = nullptr; + } + if (_old_to_region == nullptr) { + if (_empty_regions_pos < _empty_regions.length()) { + ShenandoahHeapRegion* new_to_region = _empty_regions.at(_empty_regions_pos); + _empty_regions_pos++; + new_to_region->set_affiliation(OLD_GENERATION); + _old_to_region = new_to_region; + _old_compact_point = _old_to_region->bottom(); + promote_object = true; + } + // Else this worker thread does not yet have any empty regions into which this aged object can be promoted so + // we leave promote_object as false, deferring the promotion. + } else { + promote_object = true; + } + } + + if (promote_object || (_from_affiliation == ShenandoahRegionAffiliation::OLD_GENERATION)) { + assert(_old_to_region != nullptr, "_old_to_region should not be NULL when evacuating to OLD region"); if (_old_compact_point + obj_size > _old_to_region->end()) { ShenandoahHeapRegion* new_to_region; @@ -525,12 +568,15 @@ class ShenandoahPrepareForGenerationalCompactionObjectClosure : public ObjectClo } else { assert(_from_affiliation == ShenandoahRegionAffiliation::YOUNG_GENERATION, "_from_region must be OLD_GENERATION or YOUNG_GENERATION"); - assert(_young_to_region != nullptr, "_young_to_region should not be NULL when compacting YOUNG _from_region"); + + // After full gc compaction, all regions have age 0. Embed the region's age into the object's age in order to preserve + // tenuring progress. + _heap->increase_object_age(p, from_region_age + 1); + if (_young_compact_point + obj_size > _young_to_region->end()) { ShenandoahHeapRegion* new_to_region; - log_debug(gc)("Worker %u finishing young region " SIZE_FORMAT ", compact_point: " PTR_FORMAT ", obj_size: " SIZE_FORMAT ", &compact_point[obj_size]: " PTR_FORMAT ", region end: " PTR_FORMAT, _worker_id, _young_to_region->index(), p2i(_young_compact_point), obj_size, p2i(_young_compact_point + obj_size), p2i(_young_to_region->end())); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index 9818e03f01176..40967f4407b35 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -33,6 +33,7 @@ #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp" #include "gc/shenandoah/shenandoahUtils.hpp" #include "gc/shenandoah/shenandoahVerifier.hpp" +#include "gc/shenandoah/shenandoahYoungGeneration.hpp" #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" class ShenandoahResetUpdateRegionStateClosure : public ShenandoahHeapRegionClosure { @@ -60,7 +61,7 @@ class ShenandoahResetBitmapTask : public ShenandoahHeapRegionClosure { ShenandoahMarkingContext* const _ctx; public: ShenandoahResetBitmapTask() : - _heap(ShenandoahHeap::heap()), + _heap(ShenandoahHeap::heap()), _ctx(_heap->marking_context()) {} void heap_region_do(ShenandoahHeapRegion* region) { @@ -246,7 +247,110 @@ bool ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) { ShenandoahPhaseTimings::degen_gc_choose_cset); ShenandoahHeapLocker locker(heap->lock()); heap->collection_set()->clear(); + + + if (heap->mode()->is_generational()) { + + // During initialization and phase changes, it is more likely that fewer objects die young and old-gen + // memory is not yet full (or is in the process of being replaced). During these tiems especially, it + // is beneficial to loan memory from old-gen to young-gen during the evacuation and update-refs phases + // of execution. + + // PromotionReserve for old generation: how much memory are we reserving to hold the results of + // promoting young-gen objects that have reached tenure age? This value is not "critical". If we + // underestimate, certain promotions will simply be deferred. The basis of this estimate is + // historical precedent. Conservatively, budget this value to be twice the amount of memory + // promoted in previous GC pass. Whenever the amount promoted during previous GC is zero, + // including initial passes before any objects have reached tenure age, use live memory within + // young-gen memory divided by (ShenandoahTenureAge multiplied by InitialTenuringThreshold) as the + // the very conservative value of this parameter. Note that during initialization, there is + // typically plentiful old-gen memory so it's ok to be conservative with the initial estimates + // of this value. But PromotionReserve can be no larger than available memory. In summary, we + // compute PromotionReserve as the smaller of: + // 1. old_gen->available + // 2. young_gen->capacity() * ShenandoahEvacReserve + // 3. (bytes promoted by previous promotion) * 2 if (bytes promoted by previous promotion) is not zero + // 4. if (bytes promoted by previous promotion) is zero, divide young_gen->used() + // by (ShenandoahTenureAge * InitialTenuringThreshold) + // + // We don't yet know how much live memory. Inside choose_collection_set(), after it computes live memory, + // the PromotionReserve may be further reduced. + // + // 5. live bytes in young-gen divided by (ShenandoahTenureAge * InitialTenuringThreshold + // if the number of bytes promoted by previous promotion is zero + // + ShenandoahGeneration* old_generation = heap->old_generation(); + ShenandoahYoungGeneration* young_generation = heap->young_generation(); + size_t promotion_reserve = old_generation->available(); + + size_t max_young_evacuation = (young_generation->soft_max_capacity() * ShenandoahOldEvacReserve) / 100; + if (max_young_evacuation < promotion_reserve) { + promotion_reserve = max_young_evacuation; + } + + size_t previously_promoted = heap->get_previous_promotion(); + if (previously_promoted == 0) { + // Very conservatively, assume linear population decay (rather than more typical exponential) and assume all of + // used is live. + size_t proposed_reserve = young_generation->used() / (ShenandoahAgingCyclePeriod * InitialTenuringThreshold); + if (promotion_reserve > proposed_reserve) { + promotion_reserve = proposed_reserve; + } + } else if (previously_promoted * 2 < promotion_reserve) { + promotion_reserve = previously_promoted * 2; + } + + heap->set_promotion_reserve(promotion_reserve); + heap->capture_old_usage(old_generation->used()); + + // OldEvacuationReserve for old generation: how much memory are we reserving to hold the results of + // evacuating old-gen heap regions? In order to sustain a consistent pace of young-gen collections, + // the goal is to maintain a consistent value for this parameter (when the candidate set is not + // empty). This value is the minimum of: + // 1. old_gen->available() - PromotionReserve + // 2. (young_gen->capacity() scaled by ShenandoahEvacReserve) scaled by ShenandoahOldEvacRatioPercent + + // Don't reserve for old_evac any more than the memory that is available in old_gen. + size_t old_evacuation_reserve = old_generation->available() - promotion_reserve; + + // Make sure old evacuation is no more than ShenandoahOldEvacRatioPercent of the total evacuation budget. + size_t max_total_evac = (young_generation->soft_max_capacity() * ShenandoahEvacReserve) / 100; + size_t max_old_evac_portion = (max_total_evac * ShenandoahOldEvacRatioPercent) / 100; + + if (old_evacuation_reserve > max_old_evac_portion) { + old_evacuation_reserve = max_old_evac_portion; + } + + heap->set_old_evac_reserve(old_evacuation_reserve); + heap->reset_old_evac_expended(); + + // Compute YoungEvacuationReserve after we prime the collection set with old-gen candidates. This depends + // on how much memory old-gen wants to evacuate. This is done within _heuristics->choose_collection_set(). + + // There's no need to pass this information to ShenandoahFreeSet::rebuild(). The GC allocator automatically borrows + // memory from mutator regions when necessary. + } + + // The heuristics may consult and/or change the values of PromotionReserved, OldEvacuationReserved, and + // YoungEvacuationReserved, all of which are represented in the shared ShenandoahHeap data structure. result = _heuristics->choose_collection_set(heap->collection_set(), heap->old_heuristics()); + + // EvacuationAllocationSupplement: This represents memory that can be allocated in excess of young_gen->available() + // during evacuation and update-refs. This memory can be temporarily borrowed from old-gen allotment, then + // repaid at the end of update-refs from the recycled collection set. After we have computed the collection set + // based on the parameters established above, we can make additional calculates based on our knowledge of the + // collection set to determine how much allocation we can allow during the evacuation and update-refs phases + // of execution. With full awareness of collection set, we can shrink the values of PromotionReserve, + // OldEvacuationReserve, and YoungEvacuationReserve. Then, we can compute EvacuationAllocationReserve as the + // minimum of: + // 1. old_gen->available - (PromotionReserve + OldEvacuationReserve) + // 2. The replenishment budget (number of regions in collection set - the number of regions already + // under lien for the YoungEvacuationReserve) + // + + // The possibly revised values are also consulted by the ShenandoahPacer when it establishes pacing parameters + // for evacuation and update-refs. + } { @@ -310,7 +414,7 @@ ShenandoahGeneration::ShenandoahGeneration(GenerationMode generation_mode, _ref_processor(new ShenandoahReferenceProcessor(MAX2(max_workers, 1U))), _affiliated_region_count(0), _used(0), _bytes_allocated_since_gc_start(0), _max_capacity(max_capacity), _soft_max_capacity(soft_max_capacity), - _heuristics(nullptr) { + _adjusted_capacity(soft_max_capacity), _heuristics(nullptr) { _is_marking_complete.set(); assert(max_workers > 0, "At least one queue"); for (uint i = 0; i < max_workers; ++i) { @@ -371,6 +475,20 @@ void ShenandoahGeneration::decrease_used(size_t bytes) { Atomic::sub(&_used, bytes); } +size_t ShenandoahGeneration::used_regions() const { + return _affiliated_region_count; +} + +size_t ShenandoahGeneration::free_unaffiliated_regions() const { + size_t result = soft_max_capacity() / ShenandoahHeapRegion::region_size_bytes(); + if (_affiliated_region_count > result) { + result = 0; // If old-gen is loaning regions to young-gen, affiliated regions may exceed capacity temporarily. + } else { + result -= _affiliated_region_count; + } + return result; +} + size_t ShenandoahGeneration::used_regions_size() const { return _affiliated_region_count * ShenandoahHeapRegion::region_size_bytes(); } @@ -380,3 +498,19 @@ size_t ShenandoahGeneration::available() const { size_t soft_capacity = soft_max_capacity(); return in_use > soft_capacity ? 0 : soft_capacity - in_use; } + +size_t ShenandoahGeneration::adjust_available(intptr_t adjustment) { + _adjusted_capacity = soft_max_capacity() + adjustment; + return _adjusted_capacity; +} + +size_t ShenandoahGeneration::unadjust_available() { + _adjusted_capacity = soft_max_capacity(); + return _adjusted_capacity; +} + +size_t ShenandoahGeneration::adjusted_available() const { + size_t in_use = used(); + size_t capacity = _adjusted_capacity; + return in_use > capacity ? 0 : capacity - in_use; +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index 54e8aba0bf3ff..d01170b8f9c5a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -53,6 +53,8 @@ class ShenandoahGeneration : public CHeapObj { size_t _max_capacity; size_t _soft_max_capacity; + size_t _adjusted_capacity; + ShenandoahHeuristics* _heuristics; public: ShenandoahGeneration(GenerationMode generation_mode, uint max_workers, size_t max_capacity, size_t soft_max_capacity); @@ -70,10 +72,23 @@ class ShenandoahGeneration : public CHeapObj { virtual size_t soft_max_capacity() const { return _soft_max_capacity; } virtual size_t max_capacity() const { return _max_capacity; } + virtual size_t used_regions() const; virtual size_t used_regions_size() const; + virtual size_t free_unaffiliated_regions() const; virtual size_t used() const { return _used; } virtual size_t available() const; + // During evacuation and update-refs, some memory may be shifted between generations. In particular, memory + // may be loaned by old-gen to young-gen based on the promise the loan will be promptly repaid from the memory reclaimed + // when the current collection set is recycled. The capacity adjustment also takes into consideration memory that is + // set aside within each generation to hold the results of evacuation, but not promotion, into that region. Promotions + // into old-gen are bounded by adjusted_available() whereas evacuations into old-gen are pre-committed. + virtual size_t adjusted_available() const; + + // Both of following return new value of available + virtual size_t adjust_available(intptr_t adjustment); + virtual size_t unadjust_available(); + size_t bytes_allocated_since_gc_start(); void reset_bytes_allocated_since_gc_start(); void increase_allocated(size_t bytes); @@ -143,8 +158,6 @@ class ShenandoahGeneration : public CHeapObj { void decrease_used(size_t bytes); virtual bool is_concurrent_mark_in_progress() = 0; - -private: void confirm_heuristics_mode(); }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 340d54a4e8922..77442fb1bcdea 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -502,6 +502,7 @@ ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) : _gc_generation(NULL), _mixed_evac(false), _prep_for_mixed_evac_in_progress(false), + _evacuation_allowance(0), _initial_size(0), _used(0), _committed(0), @@ -512,6 +513,14 @@ ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) : _num_regions(0), _regions(NULL), _update_refs_iterator(this), + _alloc_supplement_reserve(0), + _promotion_reserve(0), + _old_evac_reserve(0), + _old_evac_expended(0), + _young_evac_reserve(0), + _young_evac_expended(0), + _captured_old_usage(0), + _previous_promotion(0), _cancel_requested_time(0), _young_generation(NULL), _global_generation(NULL), @@ -887,7 +896,7 @@ HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) } // Establish a new PLAB and allocate size HeapWords within it. -HeapWord* ShenandoahHeap::allocate_from_plab_slow(Thread* thread, size_t size) { +HeapWord* ShenandoahHeap::allocate_from_plab_slow(Thread* thread, size_t size, bool is_promotion) { // New object should fit the PLAB size size_t min_size = MAX2(size, PLAB::min_size()); @@ -921,6 +930,8 @@ HeapWord* ShenandoahHeap::allocate_from_plab_slow(Thread* thread, size_t size) { retire_plab(plab); size_t actual_size = 0; + // allocate_new_plab resets plab_evacuated and plab_promoted and disables promotions if old-gen available is + // less than the remaining evacuation need. HeapWord* plab_buf = allocate_new_plab(min_size, new_size, &actual_size); if (plab_buf == NULL) { return NULL; @@ -942,13 +953,25 @@ HeapWord* ShenandoahHeap::allocate_from_plab_slow(Thread* thread, size_t size) { #endif // ASSERT } plab->set_buf(plab_buf, actual_size); + + if (is_promotion && !ShenandoahThreadLocalData::allow_plab_promotions(thread)) { + return nullptr; + } return plab->allocate(size); } +// TODO: It is probably most efficient to register all objects (both promotions and evacuations) that were allocated within +// this plab at the time we retire the plab. A tight registration loop will run within both code and data caches. This change +// would allow smaller and faster in-line implementation of alloc_from_plab(). Since plabs are aligned on card-table boundaries, +// this object registration loop can be performed without acquiring a lock. void ShenandoahHeap::retire_plab(PLAB* plab) { if (!mode()->is_generational()) { plab->retire(); } else { + Thread* thread = Thread::current(); + size_t evacuated = ShenandoahThreadLocalData::get_plab_evacuated(thread); + // We don't enforce limits on get_plab_promoted(thread). Promotion uses any memory not required for evacuation. + expend_old_evac(evacuated); size_t waste = plab->waste(); HeapWord* top = plab->top(); plab->retire(); @@ -995,7 +1018,7 @@ HeapWord* ShenandoahHeap::allocate_new_tlab(size_t min_size, size_t requested_size, size_t* actual_size) { ShenandoahAllocRequest req = ShenandoahAllocRequest::for_tlab(min_size, requested_size); - HeapWord* res = allocate_memory(req); + HeapWord* res = allocate_memory(req, false); if (res != NULL) { *actual_size = req.actual_size(); } else { @@ -1008,7 +1031,7 @@ HeapWord* ShenandoahHeap::allocate_new_gclab(size_t min_size, size_t word_size, size_t* actual_size) { ShenandoahAllocRequest req = ShenandoahAllocRequest::for_gclab(min_size, word_size); - HeapWord* res = allocate_memory(req); + HeapWord* res = allocate_memory(req, false); if (res != NULL) { *actual_size = req.actual_size(); } else { @@ -1021,7 +1044,9 @@ HeapWord* ShenandoahHeap::allocate_new_plab(size_t min_size, size_t word_size, size_t* actual_size) { ShenandoahAllocRequest req = ShenandoahAllocRequest::for_plab(min_size, word_size); - HeapWord* res = allocate_memory(req); + // Note that allocate_memory() sets a thread-local flag to prohibit further promotions by this thread + // if we are at risk of exceeding the old-gen evacuation budget. + HeapWord* res = allocate_memory(req, false); if (res != NULL) { *actual_size = req.actual_size(); } else { @@ -1030,7 +1055,9 @@ HeapWord* ShenandoahHeap::allocate_new_plab(size_t min_size, return res; } -HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) { +// is_promotion is true iff this allocation is known for sure to hold the result of young-gen evacuation +// to old-gen. plab allocates arre not known as such, since they may hold old-gen evacuations. +HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req, bool is_promotion) { intptr_t pacer_epoch = 0; bool in_new_region = false; HeapWord* result = NULL; @@ -1042,7 +1069,7 @@ HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) { } if (!ShenandoahAllocFailureALot || !should_inject_alloc_failure()) { - result = allocate_memory_under_lock(req, in_new_region); + result = allocate_memory_under_lock(req, in_new_region, is_promotion); } // Allocation failed, block until control thread reacted, then retry allocation. @@ -1060,18 +1087,18 @@ HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) { while (result == NULL && _progress_last_gc.is_set()) { tries++; control_thread()->handle_alloc_failure(req); - result = allocate_memory_under_lock(req, in_new_region); + result = allocate_memory_under_lock(req, in_new_region, is_promotion); } while (result == NULL && tries <= ShenandoahFullGCThreshold) { tries++; control_thread()->handle_alloc_failure(req); - result = allocate_memory_under_lock(req, in_new_region); + result = allocate_memory_under_lock(req, in_new_region, is_promotion); } } else { assert(req.is_gc_alloc(), "Can only accept GC allocs here"); - result = allocate_memory_under_lock(req, in_new_region); + result = allocate_memory_under_lock(req, in_new_region, is_promotion); // Do not call handle_alloc_failure() here, because we cannot block. // The allocation failure would be handled by the LRB slowpath with handle_alloc_failure_evac(). } @@ -1109,32 +1136,96 @@ HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req) { return result; } -HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req, bool& in_new_region) { - if (mode()->is_generational() && req.affiliation() == YOUNG_GENERATION && young_generation()->used() + req.size() >= young_generation()->max_capacity()) { - return nullptr; - } +HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req, bool& in_new_region, bool is_promotion) { + size_t requested_bytes = req.size() * HeapWordSize; ShenandoahHeapLocker locker(lock()); + if (mode()->is_generational()) { + if (req.affiliation() == YOUNG_GENERATION) { + if (req.type() == ShenandoahAllocRequest::_alloc_gclab) { + if (requested_bytes + get_young_evac_expended() > get_young_evac_reserve()) { + // This should only happen if evacuation waste is too low. Rejecting one thread's request for GCLAB does not + // necessarily result in failure of the evacuation effort. A different thread may be able to copy from-space object. + + // TODO: Should we really fail here in the case that there is sufficient memory to allow us to allocate a gclab + // beyond the young_evac_reserve? Seems it would be better to take away from mutator allocation budget if this + // prevents fall-back to full GC in order to recover from failed evacuation. + return nullptr; + } + // else, there is sufficient memory to allocate this GCLAB so do nothing here. + } else if (req.is_gc_alloc()) { + // This is a shared alloc for purposes of evacuation. + if (requested_bytes + get_young_evac_expended() > get_young_evac_reserve()) { + // TODO: Should we really fail here in the case that there is sufficient memory to allow us to allocate a gclab + // beyond the young_evac_reserve? Seems it would be better to take away from mutator allocation budget if this + // prevents fall-back to full GC in order to recover from failed evacuation. + return nullptr; + } else { + // There is sufficient memory to allocate this shared evacuation object. + } + } else if (requested_bytes >= young_generation()->adjusted_available()) { + // We know this is not a GCLAB. This must be a TLAB or a shared allocation. Reject the allocation request if + // exceeds established capacity limits. + return nullptr; + } + } else { // reg.affiliation() == OLD_GENERATION + assert(req.type() != ShenandoahAllocRequest::_alloc_gclab, "GCLAB pertains only to young-gen memory"); + + if (req.type() == ShenandoahAllocRequest::_alloc_plab) { + // We've already retired this thread's previously exhausted PLAB and have accounted for how that PLAB's + // memory was allotted. + Thread* thread = Thread::current(); + ShenandoahThreadLocalData::reset_plab_evacuated(thread); + ShenandoahThreadLocalData::reset_plab_promoted(thread); + + // Conservatively, assume this entire PLAB will be used for promotion. Act as if we need to serve the + // rest of evacuation need from as-yet unallocated old-gen memory. + size_t remaining_evac_need = get_old_evac_reserve() - get_old_evac_expended(); + size_t evac_available = old_generation()->adjusted_available() - requested_bytes; + if (remaining_evac_need >= evac_available) { + // Disable promotions within this thread because the entirety of this PLAB must be available to hold + // old-gen evacuations. + ShenandoahThreadLocalData::disable_plab_promotions(thread); + } else { + ShenandoahThreadLocalData::enable_plab_promotions(thread); + } + } else if (is_promotion) { + // This is a shared alloc for promotion + Thread* thread = Thread::current(); + size_t remaining_evac_need = get_old_evac_reserve() - get_old_evac_expended(); + size_t evac_available = old_generation()->adjusted_available() - requested_bytes; + if (remaining_evac_need >= evac_available) { + return nullptr; // We need to reserve the remaining memory for evacuation so defer the promotion + } + // Else, we'll allow the allocation to proceed. (Since we hold heap lock, the tested condition remains true.) + } else { + // This is a shared allocation for evacuation. Memory has already been reserved for this purpose. + } + } + } + HeapWord* result = _free_set->allocate(req, in_new_region); - if (result != NULL && req.affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) { - // Register the newly allocated object while we're holding the global lock since there's no synchronization - // built in to the implementation of register_object(). There are potential races when multiple independent - // threads are allocating objects, some of which might span the same card region. For example, consider - // a card table's memory region within which three objects are being allocated by three different threads: - // - // objects being "concurrently" allocated: - // [-----a------][-----b-----][--------------c------------------] - // [---- card table memory range --------------] - // - // Before any objects are allocated, this card's memory range holds no objects. Note that: - // allocation of object a wants to set the has-object, first-start, and last-start attributes of the preceding card region. - // allocation of object b wants to set the has-object, first-start, and last-start attributes of this card region. - // allocation of object c also wants to set the has-object, first-start, and last-start attributes of this card region. - // - // The thread allocating b and the thread allocating c can "race" in various ways, resulting in confusion, such as last-start - // representing object b while first-start represents object c. This is why we need to require all register_object() - // invocations to be "mutually exclusive" with respect to each card's memory range. - ShenandoahHeap::heap()->card_scan()->register_object(result); + if (result != NULL) { + if (req.affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) { + // Register the newly allocated object while we're holding the global lock since there's no synchronization + // built in to the implementation of register_object(). There are potential races when multiple independent + // threads are allocating objects, some of which might span the same card region. For example, consider + // a card table's memory region within which three objects are being allocated by three different threads: + // + // objects being "concurrently" allocated: + // [-----a------][-----b-----][--------------c------------------] + // [---- card table memory range --------------] + // + // Before any objects are allocated, this card's memory range holds no objects. Note that: + // allocation of object a wants to set the has-object, first-start, and last-start attributes of the preceding card region. + // allocation of object b wants to set the has-object, first-start, and last-start attributes of this card region. + // allocation of object c also wants to set the has-object, first-start, and last-start attributes of this card region. + // + // The thread allocating b and the thread allocating c can "race" in various ways, resulting in confusion, such as last-start + // representing object b while first-start represents object c. This is why we need to require all register_object() + // invocations to be "mutually exclusive" with respect to each card's memory range. + ShenandoahHeap::heap()->card_scan()->register_object(result); + } } return result; } @@ -1142,7 +1233,7 @@ HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req HeapWord* ShenandoahHeap::mem_allocate(size_t size, bool* gc_overhead_limit_was_exceeded) { ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared(size); - return allocate_memory(req); + return allocate_memory(req, false); } MetaWord* ShenandoahHeap::satisfy_failed_metadata_allocation(ClassLoaderData* loader_data, @@ -1385,7 +1476,7 @@ class ShenandoahRetireGCLABClosure : public ThreadClosure { void do_thread(Thread* thread) { PLAB* gclab = ShenandoahThreadLocalData::gclab(thread); assert(gclab != NULL, "GCLAB should be initialized for %s", thread->name()); - ShenandoahHeap::heap()->retire_plab(gclab); + gclab->retire(); if (_resize && ShenandoahThreadLocalData::gclab_size(thread) > 0) { ShenandoahThreadLocalData::set_gclab_size(thread, 0); } @@ -2046,6 +2137,9 @@ void ShenandoahHeap::cancel_gc(GCCause::Cause cause) { log_info(gc)("%s", msg.buffer()); Events::log(Thread::current(), "%s", msg.buffer()); _cancel_requested_time = os::elapsedTime(); + if (cause == GCCause::_shenandoah_upgrade_to_full_gc) { + _upgraded_to_full = true; + } } } @@ -2437,15 +2531,39 @@ void ShenandoahHeap::update_heap_references(bool concurrent) { class ShenandoahFinalUpdateRefsUpdateRegionStateClosure : public ShenandoahHeapRegionClosure { private: + ShenandoahMarkingContext* _ctx; ShenandoahHeapLock* const _lock; + bool _is_generational; public: - ShenandoahFinalUpdateRefsUpdateRegionStateClosure() : _lock(ShenandoahHeap::heap()->lock()) {} + ShenandoahFinalUpdateRefsUpdateRegionStateClosure( + ShenandoahMarkingContext* ctx) : _ctx(ctx), _lock(ShenandoahHeap::heap()->lock()), + _is_generational(ShenandoahHeap::heap()->mode()->is_generational()) { } void heap_region_do(ShenandoahHeapRegion* r) { + + // Maintenance of region age must follow evacuation in order to account for evacuation allocations within survivor + // regions. We consult region age during the subsequent evacuation to determine whether certain objects need to + // be promoted. + if (_is_generational && r->is_young()) { + HeapWord *tams = _ctx->top_at_mark_start(r); + HeapWord *top = r->top(); + + // Allocations move the watermark when top moves. However compacting + // objects will sometimes lower top beneath the watermark, after which, + // attempts to read the watermark will assert out (watermark should not be + // higher than top). + if (top > tams) { + // There have been allocations in this region since the start of the cycle. + // Any objects new to this region must not assimilate elevated age. + r->reset_age(); + } else if (ShenandoahHeap::heap()->is_aging_cycle()) { + r->increment_age(); + } + } + // Drop unnecessary "pinned" state from regions that does not have CP marks // anymore, as this would allow trashing them. - if (r->is_active()) { if (r->is_pinned()) { if (r->pin_count() == 0) { @@ -2472,7 +2590,7 @@ void ShenandoahHeap::update_heap_region_states(bool concurrent) { ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_update_refs_update_region_states : ShenandoahPhaseTimings::degen_gc_final_update_refs_update_region_states); - ShenandoahFinalUpdateRefsUpdateRegionStateClosure cl; + ShenandoahFinalUpdateRefsUpdateRegionStateClosure cl (young_generation()->complete_marking_context()); parallel_heap_region_iterate(&cl); assert_pinned_region_status(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index b62cde156bf29..5c41f90a34789 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -150,9 +150,12 @@ class ShenandoahHeap : public CollectedHeap { private: ShenandoahHeapLock _lock; ShenandoahGeneration* _gc_generation; - bool _mixed_evac; // true iff most recent evac included at least one old-gen HeapRegion + + bool _mixed_evac; // true iff most recent evac included at least one old-gen HeapRegion bool _prep_for_mixed_evac_in_progress; // true iff we are concurrently coalescing and filling old-gen HeapRegions + size_t _evacuation_allowance; // amount by which young-gen usage may temporarily exceed young-gen capacity + public: ShenandoahHeapLock* lock() { return &_lock; @@ -335,9 +338,67 @@ class ShenandoahHeap : public CollectedHeap { ShenandoahSharedFlag _progress_last_gc; ShenandoahSharedFlag _concurrent_strong_root_in_progress; + // _alloc_supplement_reserve is a supplemental budget for new_memory allocations. During evacuation and update-references, + // mutator allocation requests are "authorized" iff young_gen->available() plus _alloc_supplement_reserve minus + // _young_evac_reserve is greater than request size. The values of _alloc_supplement_reserve and _young_evac_reserve + // are zero except during evacuation and update-reference phases of GC. Both of these values are established at + // the start of evacuation, and they remain constant throughout the duration of these two phases of GC. Since these + // two values are constant throughout each GC phases, we introduce a new service into ShenandoahGeneration. This service + // provides adjusted_available() based on an adjusted capacity. At the start of evacuation, we adjust young capacity by + // adding the amount to be borrowed from old-gen and subtracting the _young_evac_reserve, we adjust old capacity by + // subtracting the amount to be loaned to young-gen. At the end of update-refs, we unadjust the capacity of each generation, + // and add _young_evac_expended to young-gen used. + // + // We always use adjusted capacities to determine permission to allocate within young and to promote into old. Note + // that adjusted capacities equal traditional capacities except during evacuation and update refs. + // + // During evacuation, we assure that _young_evac_expended does not exceed _young_evac_reserve and that _old_evac_expended + // does not exceed _old_evac_reserve. GCLAB allocations do not immediately affect used within the young generation + // since the adjusted capacity already accounts for the entire evacuation reserve. Each GCLAB allocations increments + // _young_evac_expended rather than incrementing the affiliated generation's used value. + // + // At the end of update references, we perform the following bookkeeping activities: + // + // 1. Unadjust the capacity within young-gen and old-gen to undo the effects of borrowing memory from old-gen. Note that + // the entirety of the collection set is now available, so allocation capacity naturally increase at this time. + // 2. Increase young_gen->used() by _young_evac_expended. This represents memory consumed by evacutions from young-gen. + // 3. Clear (reset to zero) _alloc_supplement_reserve, _young_evac_reserve, _old_evac_reserve, and _promotion_reserve + // + // _young_evac_reserve and _old_evac_reserve are only non-zero during evacuation and update-references. + // + // Allocation of young GCLABs assures that _young_evac_expended + request-size < _young_evac_reserved. If the allocation + // is authorized, increment _young_evac_expended by request size. This allocation ignores young_gen->available(). + // + // Allocation of old GCLABs assures that _old_evac_expended + request-size < _old_evac_reserved. If the allocation + // is authorized, increment _old_evac_expended by request size. This allocation ignores old_gen->available(). + // + // Note that the typical total expenditure on evacuation is less than the associated evacuation reserve because we generally + // reserve ShenandoahEvacWaste (> 1.0) times the anticipated evacuation need. In the case that there is an excessive amount + // of waste, it may be that one thread fails to grab a new GCLAB, this does not necessarily doom the associated evacuation + // effort. If this happens, the requesting thread blocks until some other thread manages to evacuate the offending object. + // Only after "all" threads fail to evacuate an object do we consider the evacuation effort to have failed. + + intptr_t _alloc_supplement_reserve; // Bytes reserved for young allocations during evac and update refs + size_t _promotion_reserve; // Bytes reserved within old-gen to hold the results of promotion + + + size_t _old_evac_reserve; // Bytes reserved within old-gen to hold evacuated objects from old-gen collection set + size_t _old_evac_expended; // Bytes of old-gen memory expended on old-gen evacuations + + size_t _young_evac_reserve; // Bytes reserved within young-gen to hold evacuated objects from young-gen collection set + size_t _young_evac_expended; // Bytes old-gen memory has been expended on young-gen evacuations + + size_t _captured_old_usage; // What was old usage (bytes) when last captured? + + size_t _previous_promotion; // Bytes promoted during previous evacuation + + bool _upgraded_to_full; + void set_gc_state_all_threads(char state); void set_gc_state_mask(uint mask, bool value); + + public: char gc_state() const; static address gc_state_addr(); @@ -372,6 +433,38 @@ class ShenandoahHeap : public CollectedHeap { inline bool is_concurrent_weak_root_in_progress() const; bool is_concurrent_prep_for_mixed_evacuation_in_progress(); inline bool is_aging_cycle() const; + inline bool upgraded_to_full() { return _upgraded_to_full; } + inline void start_conc_gc() { _upgraded_to_full = false; } + inline void record_upgrade_to_full() { _upgraded_to_full = true; } + + inline size_t capture_old_usage(size_t usage); + inline void set_previous_promotion(size_t promoted_bytes); + inline size_t get_previous_promotion() const; + + // Returns previous value + inline size_t set_promotion_reserve(size_t new_val); + inline size_t get_promotion_reserve() const; + + // Returns previous value + inline size_t set_old_evac_reserve(size_t new_val); + inline size_t get_old_evac_reserve() const; + + inline void reset_old_evac_expended(); + inline size_t expend_old_evac(size_t increment); + inline size_t get_old_evac_expended() const; + + // Returns previous value + inline size_t set_young_evac_reserve(size_t new_val); + inline size_t get_young_evac_reserve() const; + + inline void reset_young_evac_expended(); + inline size_t expend_young_evac(size_t increment); + inline size_t get_young_evac_expended() const; + + // Returns previous value. This is a signed value because it is the amount borrowed minus the amount reserved for + // young-gen evacuation. In case we cannot borrow much, this value might be negative. + inline intptr_t set_alloc_supplement_reserve(intptr_t new_val); + inline intptr_t get_alloc_supplement_reserve() const; private: void manage_satb_barrier(bool active); @@ -591,18 +684,18 @@ class ShenandoahHeap : public CollectedHeap { // ---------- Allocation support // private: - HeapWord* allocate_memory_under_lock(ShenandoahAllocRequest& request, bool& in_new_region); + HeapWord* allocate_memory_under_lock(ShenandoahAllocRequest& request, bool& in_new_region, bool is_promotion); inline HeapWord* allocate_from_gclab(Thread* thread, size_t size); HeapWord* allocate_from_gclab_slow(Thread* thread, size_t size); HeapWord* allocate_new_gclab(size_t min_size, size_t word_size, size_t* actual_size); - inline HeapWord* allocate_from_plab(Thread* thread, size_t size); - HeapWord* allocate_from_plab_slow(Thread* thread, size_t size); + inline HeapWord* allocate_from_plab(Thread* thread, size_t size, bool is_promotion); + HeapWord* allocate_from_plab_slow(Thread* thread, size_t size, bool is_promotion); HeapWord* allocate_new_plab(size_t min_size, size_t word_size, size_t* actual_size); public: - HeapWord* allocate_memory(ShenandoahAllocRequest& request); + HeapWord* allocate_memory(ShenandoahAllocRequest& request, bool is_promotion); HeapWord* mem_allocate(size_t size, bool* what); MetaWord* satisfy_failed_metadata_allocation(ClassLoaderData* loader_data, size_t size, diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp index a9364bf4ff687..d06cab21c2115 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp @@ -297,11 +297,13 @@ inline HeapWord* ShenandoahHeap::allocate_from_gclab(Thread* thread, size_t size return allocate_from_gclab_slow(thread, size); } -inline HeapWord* ShenandoahHeap::allocate_from_plab(Thread* thread, size_t size) { +inline HeapWord* ShenandoahHeap::allocate_from_plab(Thread* thread, size_t size, bool is_promotion) { assert(UseTLAB, "TLABs should be enabled"); PLAB* plab = ShenandoahThreadLocalData::plab(thread); - if (plab == NULL) { + if (is_promotion && !ShenandoahThreadLocalData::allow_plab_promotions(thread)) { + return NULL; + } else if (plab == NULL) { assert(!thread->is_Java_thread() && !thread->is_Worker_thread(), "Performance: thread should have PLAB: %s", thread->name()); // No PLABs in this thread, fallback to shared allocation @@ -309,7 +311,12 @@ inline HeapWord* ShenandoahHeap::allocate_from_plab(Thread* thread, size_t size) } HeapWord* obj = plab->allocate(size); if (obj == NULL) { - obj = allocate_from_plab_slow(thread, size); + obj = allocate_from_plab_slow(thread, size, is_promotion); + } + if (is_promotion) { + ShenandoahThreadLocalData::add_to_plab_promoted(thread, size * HeapWordSize); + } else { + ShenandoahThreadLocalData::add_to_plab_evacuated(thread, size * HeapWordSize); } return obj; } @@ -338,11 +345,12 @@ inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) { if (mark.has_displaced_mark_helper()) { // We don't want to deal with MT here just to ensure we read the right mark word. // Skip the potential promotion attempt for this one. - } else if (mark.age() >= InitialTenuringThreshold) { + } else if (r->age() + mark.age() >= InitialTenuringThreshold) { oop result = try_evacuate_object(p, thread, r, OLD_GENERATION); if (result != NULL) { return result; } + // If we failed to promote this aged object, we'll fall through to code below and evacuat to young-gen. } } return try_evacuate_object(p, thread, r, target_gen); @@ -350,10 +358,12 @@ inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) { // try_evacuate_object registers the object and dirties the associated remembered set information when evacuating // to OLD_GENERATION. -inline oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region, ShenandoahRegionAffiliation target_gen) { +inline oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region, + ShenandoahRegionAffiliation target_gen) { bool alloc_from_lab = true; HeapWord* copy = NULL; size_t size = p->size(); + bool is_promotion = (target_gen == OLD_GENERATION) && from_region->is_young(); #ifdef ASSERT if (ShenandoahOOMDuringEvacALot && @@ -365,11 +375,25 @@ inline oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, Shenandoah switch (target_gen) { case YOUNG_GENERATION: { copy = allocate_from_gclab(thread, size); + if ((copy == nullptr) && (size < ShenandoahThreadLocalData::gclab_size(thread))) { + // GCLAB allocation failed because we are bumping up against the limit on young evacuation reserve. Try resetting + // the desired GCLAB size and retry GCLAB allocation to avoid cascading of shared memory allocations. + ShenandoahThreadLocalData::set_gclab_size(thread, PLAB::min_size()); + copy = allocate_from_gclab(thread, size); + // If we still get nullptr, we'll try a shared allocation below. + } break; } case OLD_GENERATION: { if (ShenandoahUsePLAB) { - copy = allocate_from_plab(thread, size); + copy = allocate_from_plab(thread, size, is_promotion); + if ((copy == nullptr) && (size < ShenandoahThreadLocalData::plab_size(thread))) { + // PLAB allocation failed because we are bumping up against the limit on old evacuation reserve. Try resetting + // the desired PLAB size and retry PLAB allocation to avoid cascading of shared memory allocations. + ShenandoahThreadLocalData::set_plab_size(thread, PLAB::min_size()); + copy = allocate_from_plab(thread, size, is_promotion); + // If we still get nullptr, we'll try a shared allocation below. + } } break; } @@ -379,9 +403,11 @@ inline oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, Shenandoah } } } + if (copy == NULL) { + // If we failed to allocated in LAB, we'll try a shared allocation. ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared_gc(size, target_gen); - copy = allocate_memory(req); + copy = allocate_memory(req, is_promotion); alloc_from_lab = false; } #ifdef ASSERT @@ -448,6 +474,11 @@ inline oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, Shenandoah } case OLD_GENERATION: { ShenandoahThreadLocalData::plab(thread)->undo_allocation(copy, size); + if (is_promotion) { + ShenandoahThreadLocalData::subtract_from_plab_promoted(thread, size * HeapWordSize); + } else { + ShenandoahThreadLocalData::subtract_from_plab_evacuated(thread, size * HeapWordSize); + } break; } default: { @@ -561,6 +592,87 @@ inline bool ShenandoahHeap::is_aging_cycle() const { return _is_aging_cycle.is_set(); } +inline size_t ShenandoahHeap::set_promotion_reserve(size_t new_val) { + size_t orig = _promotion_reserve; + _promotion_reserve = new_val; + return orig; +} + +inline size_t ShenandoahHeap::get_promotion_reserve() const { + return _promotion_reserve; +} + +// returns previous value +size_t ShenandoahHeap::capture_old_usage(size_t old_usage) { + size_t previous_value = _captured_old_usage; + _captured_old_usage = old_usage; + return previous_value; +} + +void ShenandoahHeap::set_previous_promotion(size_t promoted_bytes) { + _previous_promotion = promoted_bytes; +} + +size_t ShenandoahHeap::get_previous_promotion() const { + return _previous_promotion; +} + +inline size_t ShenandoahHeap::set_old_evac_reserve(size_t new_val) { + size_t orig = _old_evac_reserve; + _old_evac_reserve = new_val; + return orig; +} + +inline size_t ShenandoahHeap::get_old_evac_reserve() const { + return _old_evac_reserve; +} + +inline void ShenandoahHeap::reset_old_evac_expended() { + _old_evac_expended = 0; +} + +inline size_t ShenandoahHeap::expend_old_evac(size_t increment) { + _old_evac_expended += increment; + return _old_evac_expended; +} + +inline size_t ShenandoahHeap::get_old_evac_expended() const { + return _old_evac_expended; +} + +inline size_t ShenandoahHeap::set_young_evac_reserve(size_t new_val) { + size_t orig = _young_evac_reserve; + _young_evac_reserve = new_val; + return orig; +} + +inline size_t ShenandoahHeap::get_young_evac_reserve() const { + return _young_evac_reserve; +} + +inline void ShenandoahHeap::reset_young_evac_expended() { + _young_evac_expended = 0; +} + +inline size_t ShenandoahHeap::expend_young_evac(size_t increment) { + _young_evac_expended += increment; + return _young_evac_expended; +} + +inline size_t ShenandoahHeap::get_young_evac_expended() const { + return _young_evac_expended; +} + +inline intptr_t ShenandoahHeap::set_alloc_supplement_reserve(intptr_t new_val) { + intptr_t orig = _alloc_supplement_reserve; + _alloc_supplement_reserve = new_val; + return orig; +} + +inline intptr_t ShenandoahHeap::get_alloc_supplement_reserve() const { + return _alloc_supplement_reserve; +} + template inline void ShenandoahHeap::marked_object_iterate(ShenandoahHeapRegion* region, T* cl) { marked_object_iterate(region, cl, region->top()); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index 3b10270a896a2..46d66c81b3622 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -250,7 +250,7 @@ void ShenandoahHeapRegion::make_unpinned() { void ShenandoahHeapRegion::make_cset() { shenandoah_assert_heaplocked(); - reset_age(); + // Leave age untouched. We need to consult the age when we are deciding whether to promote evacuated objects. switch (_state) { case _regular: set_state(_cset); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp index 208af1e894933..8e7ac4bec7d1c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp @@ -437,7 +437,8 @@ class ShenandoahHeapRegion { void set_affiliation(ShenandoahRegionAffiliation new_affiliation); uint age() { return _age; } - void increment_age() { if (_age < markWord::max_age) { _age++; } } + void increment_age() { _age++; } + void decrement_age() { if (_age-- == 0) { _age = 0; } } void reset_age() { _age = 0; } // Sets all remembered set cards to dirty. Returns the number of regions spanned by the associated humongous object. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp index 69c1c594cabfb..32738735f33a4 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp @@ -33,10 +33,12 @@ // If next available memory is not aligned on address that is multiple of alignment, fill the empty space // so that returned object is aligned on an address that is a multiple of alignment_in_words. Requested -// size is in words. +// size is in words. It is assumed that this->is_old(). A pad object is allocated, filled, and registered +// if necessary to assure the new allocation is properly aligned. HeapWord* ShenandoahHeapRegion::allocate_aligned(size_t size, ShenandoahAllocRequest req, size_t alignment_in_bytes) { shenandoah_assert_heaplocked_or_safepoint(); assert(is_object_aligned(size), "alloc size breaks alignment: " SIZE_FORMAT, size); + assert(is_old(), "aligned allocations are only taken from OLD regions to support PLABs"); HeapWord* obj = top(); uintptr_t addr_as_int = (uintptr_t) obj; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.cpp index 4d3f12b94f035..db9f309e6f07c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.cpp @@ -60,21 +60,6 @@ void ShenandoahFinalMarkUpdateRegionStateClosure::heap_region_do(ShenandoahHeapR } } - if (ShenandoahHeap::heap()->mode()->is_generational()) { - // Allocations move the watermark when top moves, however compacting - // objects will sometimes lower top beneath the watermark, after which, - // attempts to read the watermark will assert out (watermark should not be - // higher than top). The right way™ to check for new allocations is to compare - // top with the TAMS as is done earlier in this function. - if (top > tams) { - // There have been allocations in this region since the start of the cycle. - // Any objects new to this region must not assimilate elevated age. - r->reset_age(); - } else if (ShenandoahHeap::heap()->is_aging_cycle()) { - r->increment_age(); - } - } - // Remember limit for updating refs. It's guaranteed that we get no // from-space-refs written from here on. r->set_update_watermark_at_safepoint(r->top()); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp index 99420cfa7ec05..5166e18f19c00 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp @@ -193,7 +193,7 @@ bool ShenandoahOldGC::collect(GCCause::Cause cause) { assert(!heap->is_concurrent_strong_root_in_progress(), "No evacuations during old gc."); - vmop_entry_final_roots(); + vmop_entry_final_roots(false); if (heap->is_concurrent_prep_for_mixed_evacuation_in_progress()) { if (!entry_coalesce_and_fill()) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp b/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp index cebdaebc99bde..2a16a23c00461 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp @@ -44,6 +44,7 @@ class ShenandoahThreadLocalData { // Evacuation OOM state uint8_t _oom_scope_nesting_level; bool _oom_during_evac; + bool _plab_allows_promotion; // If false, no more promotion by this thread during this evacuation phase. SATBMarkQueue _satb_mark_queue; // Thread-local allocation buffer for object evacuations. @@ -58,6 +59,9 @@ class ShenandoahThreadLocalData { PLAB* _plab; size_t _plab_size; + size_t _plab_evacuated; + size_t _plab_promoted; + uint _worker_id; int _disarmed_value; double _paced_time; @@ -71,6 +75,8 @@ class ShenandoahThreadLocalData { _gclab_size(0), _plab(NULL), _plab_size(0), + _plab_evacuated(0), + _plab_promoted(0), _worker_id(INVALID_WORKER_ID), _disarmed_value(0), _paced_time(0) { @@ -85,6 +91,7 @@ class ShenandoahThreadLocalData { delete _gclab; } if (_plab != NULL) { + ShenandoahHeap::heap()->retire_plab(_plab); delete _plab; } } @@ -162,6 +169,50 @@ class ShenandoahThreadLocalData { data(thread)->_plab_size = v; } + static void enable_plab_promotions(Thread* thread) { + data(thread)->_plab_allows_promotion = true; + } + + static void disable_plab_promotions(Thread* thread) { + data(thread)->_plab_allows_promotion = false; + } + + static bool allow_plab_promotions(Thread* thread) { + return data(thread)->_plab_allows_promotion; + } + + static void reset_plab_evacuated(Thread* thread) { + data(thread)->_plab_evacuated = 0; + } + + static void add_to_plab_evacuated(Thread* thread, size_t increment) { + data(thread)->_plab_evacuated += increment; + } + + static void subtract_from_plab_evacuated(Thread* thread, size_t increment) { + data(thread)->_plab_evacuated -= increment; + } + + static size_t get_plab_evacuated(Thread* thread) { + return data(thread)->_plab_evacuated; + } + + static void reset_plab_promoted(Thread* thread) { + data(thread)->_plab_promoted = 0; + } + + static void add_to_plab_promoted(Thread* thread, size_t increment) { + data(thread)->_plab_promoted += increment; + } + + static void subtract_from_plab_promoted(Thread* thread, size_t increment) { + data(thread)->_plab_promoted -= increment; + } + + static size_t get_plab_promoted(Thread* thread) { + return data(thread)->_plab_promoted; + } + static void add_paced_time(Thread* thread, double v) { data(thread)->_paced_time += v; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp b/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp index 7379e8c66923f..0dbda3033d09f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp @@ -35,6 +35,7 @@ #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahReferenceProcessor.hpp" #include "gc/shenandoah/shenandoahUtils.hpp" +#include "gc/shenandoah/shenandoahYoungGeneration.hpp" #include "utilities/debug.hpp" ShenandoahPhaseTimings::Phase ShenandoahTimingsTracker::_current_phase = ShenandoahPhaseTimings::_invalid_phase; @@ -68,7 +69,14 @@ ShenandoahGCSession::ShenandoahGCSession(GCCause::Cause cause, ShenandoahGenerat ShenandoahGCSession::~ShenandoahGCSession() { + _generation->heuristics()->record_cycle_end(); + if (_heap->mode()->is_generational() && + ((_generation->generation_mode() == GLOBAL) || _heap->upgraded_to_full())) { + // If we just completed a GLOBAL GC, claim credit for completion of young-gen and old-gen GC as well + _heap->young_generation()->heuristics()->record_cycle_end(); + _heap->old_generation()->heuristics()->record_cycle_end(); + } _timer->register_gc_end(); _heap->trace_heap_after_gc(_tracer); _tracer->report_gc_reference_stats(_generation->ref_processor()->reference_process_stats()); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp index efc2a4d1bd1a7..7d4fd49d0243e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp @@ -79,5 +79,23 @@ void VM_ShenandoahFinalUpdateRefs::doit() { void VM_ShenandoahFinalRoots::doit() { ShenandoahGCPauseMark mark(_gc_id, SvcGCMarker::CONCURRENT); + if (_incr_region_ages) { + // TODO: Do we even care about this? Do we want to parallelize it? + ShenandoahHeap* heap = ShenandoahHeap::heap(); + ShenandoahMarkingContext* ctx = heap->complete_marking_context(); + + for (size_t i = 0; i < heap->num_regions(); i++) { + ShenandoahHeapRegion *r = heap->get_region(i); + if (r->is_active() && r->is_young()) { + HeapWord* tams = ctx->top_at_mark_start(r); + HeapWord* top = r->top(); + if (top > tams) { + r->reset_age(); + } else { + r->increment_age(); + } + } + } + } _gc->entry_final_roots(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp index cba55c6b78ab2..e19e020a3e120 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp @@ -134,10 +134,11 @@ class VM_ShenandoahFinalUpdateRefs: public VM_ShenandoahOperation { class VM_ShenandoahFinalRoots: public VM_ShenandoahOperation { ShenandoahConcurrentGC* const _gc; + bool _incr_region_ages; public: - VM_ShenandoahFinalRoots(ShenandoahConcurrentGC* gc) : + VM_ShenandoahFinalRoots(ShenandoahConcurrentGC* gc, bool incr_region_ages) : VM_ShenandoahOperation(), - _gc(gc) {}; + _gc(gc), _incr_region_ages(incr_region_ages) {}; VM_Operation::VMOp_Type type() const { return VMOp_ShenandoahFinalRoots; } const char* name() const { return "Shenandoah Final Roots"; } virtual void doit(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp index c952e9758807d..8a8efa1c51919 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp @@ -388,10 +388,20 @@ class ShenandoahGenerationStatsClosure : public ShenandoahHeapRegionClosure { } static void validate_usage(const char* label, ShenandoahGeneration* generation, ShenandoahCalculateRegionStatsClosure& stats) { - guarantee(stats.used() == generation->used(), + size_t generation_used; + if (generation->generation_mode() == YOUNG) { + // young_evac_expended is "usually zero". If it is non-zero, this means we are doing evacuation or updating references + // and young-gen memory that holds the results of evacuation is being temporarily hidden from the usage accounting, + // so we add it back in here to make verification happy. + generation_used = generation->used() + ShenandoahHeap::heap()->get_young_evac_expended(); + } else { + generation_used = generation->used(); + } + + guarantee(stats.used() == generation_used, "%s: generation (%s) used size must be consistent: generation-used = " SIZE_FORMAT "%s, regions-used = " SIZE_FORMAT "%s", label, generation->name(), - byte_size_in_proper_unit(generation->used()), proper_unit_for_byte_size(generation->used()), + byte_size_in_proper_unit(generation_used), proper_unit_for_byte_size(generation_used), byte_size_in_proper_unit(stats.used()), proper_unit_for_byte_size(stats.used())); } }; @@ -774,6 +784,7 @@ void ShenandoahVerifier::verify_at_safepoint(const char* label, _heap->heap_region_iterate(&cl); size_t heap_used = _heap->used(); + guarantee(cl.used() == heap_used, "%s: heap used size must be consistent: heap-used = " SIZE_FORMAT "%s, regions-used = " SIZE_FORMAT "%s", label, diff --git a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp index 1a9c2cdc83bd8..abe619aa263be 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp @@ -82,3 +82,10 @@ void ShenandoahYoungGeneration::reserve_task_queues(uint workers) { bool ShenandoahYoungGeneration::contains(oop obj) const { return ShenandoahHeap::heap()->is_in_young(obj); } + +ShenandoahHeuristics* ShenandoahYoungGeneration::initialize_heuristics(ShenandoahMode* gc_mode) { + _heuristics = gc_mode->initialize_heuristics(this); + _heuristics->set_guaranteed_gc_interval(ShenandoahGuaranteedYoungGCInterval); + confirm_heuristics_mode(); + return _heuristics; +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp index ab200bd0e5417..6fb2cb262d183 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp @@ -54,7 +54,11 @@ class ShenandoahYoungGeneration : public ShenandoahGeneration { void reserve_task_queues(uint workers) override; + virtual ShenandoahHeuristics* initialize_heuristics(ShenandoahMode* gc_mode) override; + +protected: bool is_concurrent_mark_in_progress() override; + }; #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHYOUNGGENERATION_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp index 55de158f5bbeb..ad94cf745cc99 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp @@ -102,10 +102,12 @@ range(0,100) \ \ product(uintx, ShenandoahInitFreeThreshold, 70, EXPERIMENTAL, \ - "How much heap should be free before some heuristics trigger the "\ - "initial (learning) cycles. Affects cycle frequency on startup " \ - "and after drastic state changes, e.g. after degenerated/full " \ - "GC cycles. In percents of (soft) max heap size.") \ + "When less than this amount of memory is free within the" \ + "heap or generation, trigger a learning cycle if we are " \ + "in learning mode. Learning mode happens during initialization " \ + "and following a drastic state change, such as following a " \ + "degenerated or Full GC cycle. In percents of soft max " \ + "heap size.") \ range(0,100) \ \ product(uintx, ShenandoahMinFreeThreshold, 10, EXPERIMENTAL, \ @@ -180,6 +182,11 @@ "Heuristics may trigger collections more frequently. Time is in " \ "milliseconds. Setting this to 0 disables the feature.") \ \ + product(uintx, ShenandoahGuaranteedYoungGCInterval, 5*60*1000, EXPERIMENTAL, \ + "Run a collection of the young generation at least this often. " \ + "Heuristics may trigger collections more frequently. Time is in " \ + "milliseconds. Setting this to 0 disables the feature.") \ + \ product(bool, ShenandoahAlwaysClearSoftRefs, false, EXPERIMENTAL, \ "Unconditionally clear soft references, instead of using any " \ "other cleanup policy. This minimizes footprint at expense of" \ @@ -255,17 +262,28 @@ "if in generational mode and UseTLAB is also set.") \ \ product(uintx, ShenandoahEvacReserve, 5, EXPERIMENTAL, \ - "How much of heap to reserve for evacuations. Larger values make "\ - "GC evacuate more live objects on every cycle, while leaving " \ - "less headroom for application to allocate in. In percents of " \ - "total heap size.") \ + "How much of (young-generation) heap to reserve for " \ + "(young-generation) evacuations. Larger values allow GC to " \ + "evacuate more live objects on every cycle, while leaving " \ + "less headroom for application to allocate while GC is " \ + "evacuating and updating references. This parameter is " \ + "consulted at the of marking, before selecting the collection " \ + "set. If available memory at this time is smaller than the " \ + "indicated reserve, the bound on collection set size is " \ + "adjusted downward. The size of a generational mixed " \ + "evacuation collection set (comprised of both young and old " \ + "regions) is also bounded by this parameter. In percents of " \ + "total (young-generation) heap size.") \ range(1,100) \ \ product(double, ShenandoahEvacWaste, 1.2, EXPERIMENTAL, \ "How much waste evacuations produce within the reserved space. " \ "Larger values make evacuations more resilient against " \ "evacuation conflicts, at expense of evacuating less on each " \ - "GC cycle.") \ + "GC cycle. Smaller values increase the risk of evacuation " \ + "failures, which will trigger stop-the-world Full GC passes. " \ + "A minimum value of 1.6 is recommended for Generational " \ + "mode of Shenandoah.") \ range(1.0,100.0) \ \ product(bool, ShenandoahEvacReserveOverflow, true, EXPERIMENTAL, \ @@ -274,12 +292,39 @@ "reserve/waste is incorrect, at the risk that application " \ "runs out of memory too early.") \ \ - product(double, ShenandoahOldEvacReserve, 5.0, EXPERIMENTAL, \ - "How much of old generation to withhold from evacuations. " \ - "Larger values will result in fewer live objects being " \ - "evacuated in the old generation.") \ + product(uintx, ShenandoahOldEvacReserve, 2, EXPERIMENTAL, \ + "How much of old-generation heap to reserve for old-generation " \ + "evacuations. Larger values allow GC to evacuate more live " \ + "old-generation objects on every cycle, while potentially " \ + "creating greater impact on the cadence at which the young- " \ + "generation allocation pool is replenished. During mixed " \ + "evacuations, the bound on amount of old-generation heap " \ + "regions included in the collecdtion set is the smaller " \ + "of the quantities specified by this parameter and the " \ + "size of ShenandoahEvacReserve as adjusted by the value of " \ + "ShenandoahOldEvacRatioPercent. In percents of total " \ + "old-generation heap size.") \ range(1,100) \ \ + product(uintx, ShenandoahOldEvacRatioPercent, 12, EXPERIMENTAL, \ + "The maximum proportion of evacuation from old-gen memory, as " \ + "a percent ratio. The default value 12 denotes that no more " \ + "than one eighth (12%) of the collection set evacuation " \ + "workload may be comprised of old-gen heap regions. A larger " \ + "value allows a smaller number of mixed evacuations to process " \ + "the entire list of old-gen collection candidates at the cost " \ + "of an increased disruption of the normal cadence of young-gen " \ + "collections. A value of 100 allows a mixed evacuation to " \ + "focus entirely on old-gen memory, allowing no young-gen " \ + "regions to be collected, likely resulting in subsequent " \ + "allocation failures because the allocation pool is not " \ + "replenished. A value of 0 allows a mixed evacuation to" \ + "focus entirely on young-gen memory, allowing no old-gen " \ + "regions to be collected, likely resulting in subsequent " \ + "promotion failures and triggering of stop-the-world full GC " \ + "events.") \ + range(0,100) \ + \ product(bool, ShenandoahPacing, true, EXPERIMENTAL, \ "Pace application allocations to give GC chance to start " \ "and complete before allocation failure is reached.") \ @@ -405,22 +450,35 @@ "Fix references with load reference barrier. Disabling this " \ "might degrade performance.") \ \ - product(uintx, ShenandoahTenuredRegionUsageBias, 192, EXPERIMENTAL, \ + product(uintx, ShenandoahTenuredRegionUsageBias, 16, EXPERIMENTAL, \ "The collection set is comprised of heap regions that contain " \ "the greatest amount of garbage. " \ "For purposes of selecting regions to be included in the " \ "collection set, regions that have reached the tenure age will " \ - "be treated as if their contained garbage is the actual " \ - "contained garbage multiplied by " \ - "(ShenandoahTenuredRegionUsageBias / 128 (e.g. 150% for the " \ - "default value) as many times as the region's age meets or " \ - "exceeds the tenure age. For example, if tenure age is 7, " \ + "be treated as if their contained garbage is the contained " \ + "garbage multiplied by ShenandoahTenuredRegionUsageBias as " \ + "many times as the age of the region meets or exceeds " \ + "tenure age. For example, if tenure age is 7, " \ "the region age is 9, ShenandoahTenuredRegionUsageBias is " \ - "192, and the region is 12.5% garbage, this region " \ + "16, and the region is 12.5% garbage, this region " \ "will by treated as if its garbage content is " \ - "1/8 * 27/8 = 42.2% when comparing this region to untenured " \ - "regions.") \ - range(128, 256) \ + "12.5% * 16 * 16 * 16 = 51,200% when comparing this region " \ + " to untenured regions.") \ + range(1,128) \ + \ + product(uintx, ShenandoahBorrowPercent, 30, EXPERIMENTAL, \ + "During evacuation and reference updating in generational " \ + "mode, new allocations are allowed to borrow from old-gen " \ + "memory up to ShenandoahBorrowPercent / 100 amount of the " \ + "young-generation content of the current collection set. " \ + "Any memory borrowed from old-gen during evacuation and " \ + "update-references phases of GC will be repaid from the " \ + "abundance of young-gen memory produced when the collection " \ + "set is recycled at the end of updating references. The " \ + "default value of 30 reserves 70% of the to-be-reclaimed " \ + "young collection set memory to be allocated during the " \ + "subsequent concurrent mark phase of GC.") \ + range(0, 100) \ \ product(bool, ShenandoahPromoteTenuredObjects, true, DIAGNOSTIC, \ "Turn on/off evacuating individual tenured young objects " \ @@ -435,7 +493,4 @@ "regions each time this many young-gen GC cycles are completed.") // end of GC_SHENANDOAH_FLAGS -// 2^ShenandoahTenuredRegionUsageBiasLogBase2 is 128 -#define ShenandoahTenuredRegionUsageBiasLogBase2 7 - #endif // SHARE_GC_SHENANDOAH_SHENANDOAH_GLOBALS_HPP From dc9d7d0ab4a5acc144ab4bba9d4608a29e96cac4 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Tue, 8 Feb 2022 18:44:23 +0000 Subject: [PATCH 106/254] Fix two errors in recent commit Reviewed-by: wkemper --- .../gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp | 2 +- src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp index c9886ffb9abd5..d5302ef872370 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp @@ -283,7 +283,7 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { // pass. The best prediction for this aspect of spikes in allocation patterns is probably recent past history. // // Rationale: - // The idea is that there is an average allocation rate and there are occassional abnormal bursts (or spikes) of + // The idea is that there is an average allocation rate and there are occasional abnormal bursts (or spikes) of // allocations that exceed the average allocation rate. What do these spikes look like? // // 1. At certain phase changes, we may discard large amounts of data and replace it with large numbers of newly diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index 881f784418394..40e33bb0d214e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -299,7 +299,7 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah } else { assert(r->affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION, "GC Alloc was not YOUNG so must be OLD"); - assert(req.type() != _alloc_gclab, "old-gen allocations use PLAB or shared allocation"); + assert(req.type() != ShenandoahAllocRequest::_alloc_gclab, "old-gen allocations use PLAB or shared allocation"); _heap->old_generation()->increase_used(size * HeapWordSize); // for plabs, we'll sort the difference between evac and promotion usage when we retire the plab } From cd0d9c191774eed4b35d9292fa608d45f2effea8 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Wed, 9 Feb 2022 23:30:19 +0000 Subject: [PATCH 107/254] Use active generation, rather than young exclusively when preparing for update refs Reviewed-by: kdnilsen --- src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 77442fb1bcdea..9571ce04e739d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -2590,7 +2590,7 @@ void ShenandoahHeap::update_heap_region_states(bool concurrent) { ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_update_refs_update_region_states : ShenandoahPhaseTimings::degen_gc_final_update_refs_update_region_states); - ShenandoahFinalUpdateRefsUpdateRegionStateClosure cl (young_generation()->complete_marking_context()); + ShenandoahFinalUpdateRefsUpdateRegionStateClosure cl (active_generation()->complete_marking_context()); parallel_heap_region_iterate(&cl); assert_pinned_region_status(); From 47ca07c33fd56410be46304bd8957be07edc0859 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Thu, 10 Feb 2022 18:09:32 +0000 Subject: [PATCH 108/254] Fixes for running in non-generational modes Reviewed-by: kdnilsen --- .../share/gc/shenandoah/shenandoahDegeneratedGC.cpp | 2 +- src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp index b633ea5dcc1f1..98a5ff3006a10 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp @@ -118,7 +118,7 @@ void ShenandoahDegenGC::op_degenerated() { // changing the cycle parameters mid-cycle during concurrent -> degenerated handover. heap->set_unload_classes((!heap->mode()->is_generational() || _generation->generation_mode() == GLOBAL) && _generation->heuristics()->can_unload_classes()); - if (_generation->generation_mode() == YOUNG || (_generation->generation_mode() == GLOBAL && ShenandoahVerify)) { + if (heap->mode()->is_generational() && (_generation->generation_mode() == YOUNG || (_generation->generation_mode() == GLOBAL && ShenandoahVerify))) { // Swap remembered sets for young, or if the verifier will run during a global collect _generation->swap_remembered_set(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index 40e33bb0d214e..a3cb6a9928880 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -295,7 +295,12 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah // This is either a GCLAB or it is a shared evacuation allocation. In either case, we expend young evac. // At end of update refs, we'll add expended young evac into young_gen->used. We hide this usage // from current accounting because memory reserved for evacuation is not part of adjusted capacity. - _heap->expend_young_evac(size * HeapWordSize); + if (_heap->mode()->is_generational()) { + _heap->expend_young_evac(size * HeapWordSize); + } else { + // If we are not in generational mode, we still need to count this allocation as used memory. + _heap->young_generation()->increase_used(size * HeapWordSize); + } } else { assert(r->affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION, "GC Alloc was not YOUNG so must be OLD"); From a9a9f138d88a31054db2c4ae169c26f05ebd2da5 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Tue, 22 Feb 2022 00:13:59 +0000 Subject: [PATCH 109/254] Coalesce dead objects in pinned old regions during full gc Reviewed-by: kdnilsen --- src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index 8a8c3e4ad4287..7286ce9dbde2a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -1125,6 +1125,13 @@ class ShenandoahAdjustPointersTask : public WorkerTask { if (!r->is_humongous_continuation() && r->has_live()) { _heap->marked_object_iterate(r, &obj_cl); } + if (r->is_pinned() && r->is_old() && r->is_active() && !r->is_humongous()) { + // Pinned regions are not compacted so they may still hold unmarked objects with + // reference to reclaimed memory. Remembered set scanning will crash if it attempts + // to iterate the oops in these objects. + r->begin_preemptible_coalesce_and_fill(); + r->oop_fill_and_coalesce(); + } r = _regions.next(); } } From 2132a6d151e4c46b71a220100fa478a8be676f13 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Tue, 22 Feb 2022 00:27:05 +0000 Subject: [PATCH 110/254] Mutator may fail to evacuate Reviewed-by: wkemper --- .../share/gc/shenandoah/shenandoahHeap.cpp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 169d63d2e8d82..ffe1fc89003a7 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -2111,10 +2111,18 @@ bool ShenandoahHeap::try_cancel_gc() { assert(prev == NOT_CANCELLED, "must be NOT_CANCELLED"); Thread* thread = Thread::current(); if (thread->is_Java_thread()) { - // We need to provide a safepoint here, otherwise we might - // spin forever if a SP is pending. - ThreadBlockInVM sp(JavaThread::cast(thread)); - SpinPause(); + JavaThread* java_thread = JavaThread::cast(thread); + if (java_thread->thread_state() == _thread_in_Java) { + // ThreadBlockInVM requires thread state to be _thread_in_vm. If we are in Java, safely transition thread state. + ThreadInVMfromJava transition(java_thread); + // We need to provide a safepoint here. Otherwise we might spin forever if a SP is pending. + ThreadBlockInVM sp(JavaThread::cast(thread)); + SpinPause(); + } else { + // We need to provide a safepoint here. Otherwise we might spin forever if a SP is pending. + ThreadBlockInVM sp(JavaThread::cast(thread)); + SpinPause(); + } } } } From 0a57127c34b133bfea61cbd67a16a958bc1f4377 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Wed, 23 Feb 2022 23:57:55 +0000 Subject: [PATCH 111/254] Clear cards for old regions when they are put into service Reviewed-by: kdnilsen --- .../share/gc/shenandoah/shenandoahFreeSet.cpp | 20 +++++++++---------- .../gc/shenandoah/shenandoahHeapRegion.cpp | 2 -- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index 47a4862f1141b..be6d9882c21e4 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -177,16 +177,17 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah ShenandoahMarkingContext* const ctx = _heap->complete_marking_context(); r->set_affiliation(req.affiliation()); - r->set_update_watermark(r->bottom()); - // Any OLD region allocated during concurrent coalesce-and-fill does not need to be coalesced and filled because - // all objects allocated within this region are above TAMS (and thus are implicitly marked). In case this is an - // OLD region and concurrent preparation for mixed evacuations visits this region before the start of the next - // old-gen concurrent mark (i.e. this region is allocated following the start of old-gen concurrent mark but before - // concurrent preparations for mixed evacuations are completed), we mark this region as not requiring any - // coalesce-and-fill processing. This code is only necessary if req.affiliation() is OLD, but harmless if not. - r->end_preemptible_coalesce_and_fill(); - ctx->capture_top_at_mark_start(r); + if (r->is_old()) { + // Any OLD region allocated during concurrent coalesce-and-fill does not need to be coalesced and filled because + // all objects allocated within this region are above TAMS (and thus are implicitly marked). In case this is an + // OLD region and concurrent preparation for mixed evacuations visits this region before the start of the next + // old-gen concurrent mark (i.e. this region is allocated following the start of old-gen concurrent mark but before + // concurrent preparations for mixed evacuations are completed), we mark this region as not requiring any + // coalesce-and-fill processing. + r->end_preemptible_coalesce_and_fill(); + _heap->clear_cards_for(r); + } assert(ctx->top_at_mark_start(r) == r->bottom(), "Newly established allocation region starts with TAMS equal to bottom"); assert(ctx->is_bitmap_clear_range(ctx->top_bitmap(r), r->end()), "Bitmap above top_bitmap() must be clear"); @@ -508,7 +509,6 @@ void ShenandoahFreeSet::try_recycle_trashed(ShenandoahHeapRegion *r) { void ShenandoahFreeSet::recycle_trash() { // lock is not reentrable, check we don't have it shenandoah_assert_not_heaplocked(); - for (size_t i = 0; i < _heap->num_regions(); i++) { ShenandoahHeapRegion* r = _heap->get_region(i); if (r->is_trash()) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index 46d66c81b3622..127f4037d0674 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -569,8 +569,6 @@ void ShenandoahHeapRegion::recycle() { make_empty(); set_affiliation(FREE); - heap->clear_cards_for(this); - if (ZapUnusedHeapArea) { SpaceMangler::mangle_region(MemRegion(bottom(), end())); } From b54dca1e029135985509e926346b38d3ee0a0c3c Mon Sep 17 00:00:00 2001 From: William Kemper Date: Tue, 1 Mar 2022 00:12:41 +0000 Subject: [PATCH 112/254] Remove young generation access to old generation mark queues earlier Reviewed-by: kdnilsen --- src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp | 3 --- src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp | 5 +++++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index e16387f18e1c3..5d1e09b6c76fc 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -525,9 +525,6 @@ void ShenandoahControlThread::resume_concurrent_old_cycle(ShenandoahGeneration* // is allowed to cancel a GC. ShenandoahOldGC gc(generation, _allow_old_preemption); if (gc.collect(cause)) { - // Old collection is complete, the young generation no longer needs this - // reference to the old concurrent mark so clean it up. - heap->young_generation()->set_old_gen_task_queues(NULL); generation->heuristics()->record_success_concurrent(); heap->shenandoah_policy()->record_success_concurrent(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp index d9a3f17c51085..ac12ff3d215d4 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp @@ -30,6 +30,7 @@ #include "gc/shenandoah/shenandoahOldGC.hpp" #include "gc/shenandoah/shenandoahOopClosures.inline.hpp" #include "gc/shenandoah/shenandoahGeneration.hpp" +#include "gc/shenandoah/shenandoahYoungGeneration.hpp" #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" #include "gc/shenandoah/shenandoahWorkerPolicy.hpp" #include "prims/jvmtiTagMap.hpp" @@ -105,6 +106,10 @@ void ShenandoahOldGC::op_final_mark() { _mark.finish_mark(); assert(!heap->cancelled_gc(), "STW mark cannot OOM"); + // Old collection is complete, the young generation no longer needs this + // reference to the old concurrent mark so clean it up. + heap->young_generation()->set_old_gen_task_queues(NULL); + // We need to do this because weak root cleaning reports the number of dead handles JvmtiTagMap::set_needs_cleaning(); From 98b21327d650326486daf5db2ffdf8fd8dde59b8 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Wed, 2 Mar 2022 17:59:55 +0000 Subject: [PATCH 113/254] Reset update watermark for cycles degenerated during evacuation Reviewed-by: kdnilsen --- .../gc/shenandoah/shenandoahDegeneratedGC.cpp | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp index 423ff654fd3be..5a38a0e600f9d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp @@ -167,6 +167,27 @@ void ShenandoahDegenGC::op_degenerated() { // and we can do evacuation. Otherwise, it would be the shortcut cycle. if (heap->is_evacuation_in_progress()) { + if (_degen_point == _degenerated_evac) { + // Degeneration under oom-evac protocol allows the mutator LRB to expose + // references to from-space objects. This is okay, in theory, because we + // will come to the safepoint here to complete the evacuations and update + // the references. However, if the from-space reference is written to a + // region that was EC during final mark or was recycled after final mark + // it will not have TAMS or UWM updated. Such a region is effectively + // skipped during update references which can lead to crashes and corruption + // if the from-space reference is accessed. + if (UseTLAB) { + heap->labs_make_parsable(); + } + + for (size_t i = 0; i < heap->num_regions(); i++) { + ShenandoahHeapRegion* r = heap->get_region(i); + if (r->is_active() && r->top() > r->get_update_watermark()) { + r->set_update_watermark_at_safepoint(r->top()); + } + } + } + // Degeneration under oom-evac protocol might have left some objects in // collection set un-evacuated. Restart evacuation from the beginning to // capture all objects. For all the objects that are already evacuated, From a2b5be26bfc85b4c445f82697856bef8bd6175f6 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Fri, 4 Mar 2022 22:32:56 +0000 Subject: [PATCH 114/254] Age objects before installing forwarding pointer to them Reviewed-by: kdnilsen --- .../gc/shenandoah/shenandoahHeap.inline.hpp | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp index e5c6de6fd62d7..f47be76d23fc8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp @@ -440,20 +440,16 @@ inline oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, Shenandoah oop copy_val = cast_to_oop(copy); + if (mode()->is_generational() && target_gen == YOUNG_GENERATION && is_aging_cycle()) { + ShenandoahHeap::increase_object_age(copy_val, from_region->age() + 1); + } + // Try to install the new forwarding pointer. oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val); if (result == copy_val) { // Successfully evacuated. Our copy is now the public one! - if (mode()->is_generational()) { - if (target_gen == OLD_GENERATION) { - handle_old_evacuation(copy, size, from_region->is_young()); - } else if (target_gen == YOUNG_GENERATION) { - if (is_aging_cycle()) { - ShenandoahHeap::increase_object_age(copy_val, from_region->age() + 1); - } - } else { - ShouldNotReachHere(); - } + if (mode()->is_generational() && target_gen == OLD_GENERATION) { + handle_old_evacuation(copy, size, from_region->is_young()); } shenandoah_assert_correct(NULL, copy_val); return copy_val; From 958ea3bc0b42f6a1df05dab1f10498788ad73d2d Mon Sep 17 00:00:00 2001 From: William Kemper Date: Wed, 16 Mar 2022 20:36:59 +0000 Subject: [PATCH 115/254] Allow old collections to span degenerated young collections Reviewed-by: kdnilsen --- .../gc/shenandoah/shenandoahConcurrentGC.cpp | 34 ++++++++------- .../gc/shenandoah/shenandoahControlThread.cpp | 16 ++++--- .../gc/shenandoah/shenandoahDegeneratedGC.cpp | 43 +++++++++++++++---- .../share/gc/shenandoah/shenandoahFullGC.cpp | 5 +++ .../gc/shenandoah/shenandoahGeneration.cpp | 5 +-- .../gc/shenandoah/shenandoahGeneration.hpp | 4 +- .../shenandoah/shenandoahGlobalGeneration.cpp | 16 +------ .../shenandoah/shenandoahGlobalGeneration.hpp | 2 - .../share/gc/shenandoah/shenandoahHeap.cpp | 39 ++++++++++------- .../share/gc/shenandoah/shenandoahHeap.hpp | 5 ++- .../gc/shenandoah/shenandoahHeapRegion.cpp | 2 +- .../gc/shenandoah/shenandoahMark.inline.hpp | 6 ++- .../shenandoahMarkingContext.inline.hpp | 2 +- .../share/gc/shenandoah/shenandoahOldGC.cpp | 25 ++++++----- .../gc/shenandoah/shenandoahOldGeneration.cpp | 30 +++++++------ .../gc/shenandoah/shenandoahOldGeneration.hpp | 4 +- 16 files changed, 139 insertions(+), 99 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index 8f4ce81b43240..6c607b1ad5526 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -309,14 +309,6 @@ void ShenandoahConcurrentGC::entry_init_mark() { ShenandoahWorkerPolicy::calc_workers_for_init_marking(), "init marking"); - if (ShenandoahHeap::heap()->mode()->is_generational() - && (_generation->generation_mode() == YOUNG || (_generation->generation_mode() == GLOBAL && ShenandoahVerify))) { - // The current implementation of swap_remembered_set() copies the write-card-table - // to the read-card-table. The remembered sets are also swapped for GLOBAL collections - // so that the verifier works with the correct copy of the card table when verifying. - _generation->swap_remembered_set(); - } - op_init_mark(); } @@ -607,6 +599,25 @@ void ShenandoahConcurrentGC::op_init_mark() { assert(!_generation->is_mark_complete(), "should not be complete"); assert(!heap->has_forwarded_objects(), "No forwarded objects on this path"); + + if (heap->mode()->is_generational()) { + if (_generation->generation_mode() == YOUNG || (_generation->generation_mode() == GLOBAL && ShenandoahVerify)) { + // The current implementation of swap_remembered_set() copies the write-card-table + // to the read-card-table. The remembered sets are also swapped for GLOBAL collections + // so that the verifier works with the correct copy of the card table when verifying. + _generation->swap_remembered_set(); + } + + if (_generation->generation_mode() == GLOBAL) { + heap->cancel_old_gc(); + } else if (heap->is_concurrent_old_mark_in_progress()) { + // Purge the SATB buffers, transferring any valid, old pointers to the + // old generation mark queue. Any pointers in a young region will be + // abandoned. + heap->transfer_old_pointers_from_satb(); + } + } + if (ShenandoahVerify) { heap->verifier()->verify_before_concmark(); } @@ -1131,13 +1142,6 @@ void ShenandoahConcurrentGC::op_final_updaterefs() { heap->update_heap_region_states(true /*concurrent*/); - if (heap->is_concurrent_old_mark_in_progress()) { - // Purge the SATB buffers, transferring any valid, old pointers to the - // old generation mark queue. From here on, no mutator will have access - // to anything that will be trashed and recycled. - heap->purge_old_satb_buffers(false /* abandon */); - } - heap->set_update_refs_in_progress(false); heap->set_has_forwarded_objects(false); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index 5d1e09b6c76fc..48d4789522cf2 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -467,7 +467,11 @@ void ShenandoahControlThread::service_concurrent_old_cycle(const ShenandoahHeap* young_generation->set_mark_incomplete(); old_generation->set_mark_incomplete(); service_concurrent_cycle(young_generation, cause, true); - if (!heap->cancelled_gc()) { + if (heap->cancelled_gc()) { + // Young generation bootstrap cycle has failed. Concurrent mark for old generation + // is not going to resume after degenerated young cycle completes. + log_info(gc)("Bootstrap cycle for old generation was cancelled."); + } else { // Reset the degenerated point. Normally this would happen at the top // of the control loop, but here we have just completed a young cycle // which has bootstrapped the old concurrent marking. @@ -663,15 +667,13 @@ bool ShenandoahControlThread::service_stw_degenerated_cycle(GCCause::Cause cause ShenandoahGCSession session(cause, _degen_generation); ShenandoahDegenGC gc(point, _degen_generation); - - // Just in case degenerated cycle preempted old-gen marking, clear the old-gen task queues. - heap->young_generation()->set_old_gen_task_queues(NULL); - gc.collect(cause); assert(heap->young_generation()->task_queues()->is_empty(), "Unexpected young generation marking tasks"); - assert(heap->old_generation()->task_queues()->is_empty(), "Unexpected old generation marking tasks"); - assert(heap->global_generation()->task_queues()->is_empty(), "Unexpected global generation marking tasks"); + if (_degen_generation->generation_mode() == GLOBAL) { + assert(heap->old_generation()->task_queues()->is_empty(), "Unexpected old generation marking tasks"); + assert(heap->global_generation()->task_queues()->is_empty(), "Unexpected global generation marking tasks"); + } _degen_generation->heuristics()->record_success_degenerated(); heap->shenandoah_policy()->record_success_degenerated(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp index 5a38a0e600f9d..9fafda229cc43 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp @@ -90,13 +90,21 @@ void ShenandoahDegenGC::op_degenerated() { // some phase, we have to upgrade the Degenerate GC to Full GC. heap->clear_cancelled_gc(true /* clear oom handler */); - // We can't easily clear the old mark in progress flag because it must be done - // on a safepoint (not sure if that is a hard requirement). At any rate, once - // we are in a degenerated cycle, there should be no more old marking. - if (heap->is_concurrent_old_mark_in_progress()) { - heap->old_generation()->cancel_marking(); +#ifdef ASSERT + if (heap->mode()->is_generational()) { + if (_generation->generation_mode() == GenerationMode::GLOBAL) { + // We can only get to a degenerated global cycle _after_ a concurrent global cycle + // has been cancelled. In which case, we expect the concurrent global cycle to have + // cancelled the old gc already. + assert(!heap->is_old_gc_active(), "Old GC should not be active during global cycle."); + } + + if (!heap->is_concurrent_old_mark_in_progress()) { + // If we are not marking the old generation, there should be nothing in the old mark queues + assert(heap->old_generation()->task_queues()->is_empty(), "Old gen task queues should be empty."); + } } - assert(heap->old_generation()->task_queues()->is_empty(), "Old gen task queues should be empty."); +#endif ShenandoahMetricsSnapshot metrics; metrics.snap_before(); @@ -113,6 +121,14 @@ void ShenandoahDegenGC::op_degenerated() { // we can do the most aggressive degen cycle, which includes processing references and // class unloading, unless those features are explicitly disabled. + if (heap->is_concurrent_old_mark_in_progress()) { + // We have come straight into a degenerated cycle without running a concurrent cycle + // first and the SATB barrier is enabled to support concurrent old marking. The SATB buffer + // may hold a mix of old and young pointers. The old pointers need to be transferred + // to the old generation mark queues and the young pointers are _not_ part of this + // snapshot, so they must be dropped here. + heap->transfer_old_pointers_from_satb(); + } // Note that we can only do this for "outside-cycle" degens, otherwise we would risk // changing the cycle parameters mid-cycle during concurrent -> degenerated handover. @@ -125,8 +141,17 @@ void ShenandoahDegenGC::op_degenerated() { case _degenerated_roots: // Degenerated from concurrent root mark, reset the flag for STW mark - if (heap->is_concurrent_mark_in_progress()) { - heap->cancel_concurrent_mark(); + if (!heap->mode()->is_generational()) { + if (heap->is_concurrent_mark_in_progress()) { + heap->cancel_concurrent_mark(); + } + } else { + if (_generation->is_concurrent_mark_in_progress()) { + // We want to allow old generation marking to be punctuated by young collections + // (even if they have degenerated). If this is a global cycle, we'd have cancelled + // the entire old gc before coming into this switch. + _generation->cancel_marking(); + } } if (_degen_point == ShenandoahDegenPoint::_degenerated_roots) { @@ -293,7 +318,7 @@ void ShenandoahDegenGC::op_reset() { } void ShenandoahDegenGC::op_mark() { - assert(!ShenandoahHeap::heap()->is_concurrent_mark_in_progress(), "Should be reset"); + assert(!_generation->is_concurrent_mark_in_progress(), "Should be reset"); ShenandoahGCPhase phase(ShenandoahPhaseTimings::degen_gc_stw_mark); ShenandoahSTWMark mark(_generation, false /*full gc*/); mark.mark(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index 7286ce9dbde2a..25349704d62ad 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -194,6 +194,11 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) { heap->reset_old_evac_expended(); heap->set_promotion_reserve(0); + if (heap->mode()->is_generational()) { + // Full GC supersedes any marking or coalescing in old generation. + heap->cancel_old_gc(); + } + if (ShenandoahVerify) { heap->verifier()->verify_before_fullgc(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index 40967f4407b35..f6f365410945e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -397,12 +397,11 @@ ShenandoahMarkingContext* ShenandoahGeneration::complete_marking_context() { void ShenandoahGeneration::cancel_marking() { if (is_concurrent_mark_in_progress()) { - set_concurrent_mark_in_progress(false); + set_mark_incomplete(); } - set_mark_incomplete(); _task_queues->clear(); - ref_processor()->abandon_partial_discovery(); + set_concurrent_mark_in_progress(false); } ShenandoahGeneration::ShenandoahGeneration(GenerationMode generation_mode, diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index d01170b8f9c5a..5669ffc12bdf4 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -109,13 +109,13 @@ class ShenandoahGeneration : public CHeapObj { void merge_write_table(); // Used by concurrent and degenerated GC to reset regions. - virtual void prepare_gc(bool do_old_gc_bootstrap); + void prepare_gc(bool do_old_gc_bootstrap); // Return true iff prepared collection set includes at least one old-gen HeapRegion. virtual bool prepare_regions_and_collection_set(bool concurrent); // Cancel marking (used by Full collect and when cancelling cycle). - void cancel_marking(); + virtual void cancel_marking(); // Return true if this region is affiliated with this generation. virtual bool contains(ShenandoahHeapRegion* region) const = 0; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp index b4a22df2f1bc7..a7299fab1864b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp @@ -58,14 +58,12 @@ size_t ShenandoahGlobalGeneration::available() const { void ShenandoahGlobalGeneration::set_concurrent_mark_in_progress(bool in_progress) { ShenandoahHeap* heap = ShenandoahHeap::heap(); - if (in_progress && heap->is_concurrent_old_mark_in_progress()) { + if (in_progress && heap->mode()->is_generational()) { // Global collection has preempted an old generation mark. This is fine // because the global generation includes the old generation, but we // want the global collect to start from a clean slate and we don't want // any stale state in the old generation. - heap->purge_old_satb_buffers(true /* abandon */); - heap->old_generation()->cancel_marking(); - heap->young_generation()->set_old_gen_task_queues(nullptr); + heap->cancel_old_gc(); } heap->set_concurrent_young_mark_in_progress(in_progress); @@ -88,13 +86,3 @@ bool ShenandoahGlobalGeneration::is_concurrent_mark_in_progress() { ShenandoahHeap* heap = ShenandoahHeap::heap(); return heap->is_concurrent_mark_in_progress(); } - -void ShenandoahGlobalGeneration::prepare_gc(bool do_old_gc_bootstrap) { - ShenandoahGeneration::prepare_gc(do_old_gc_bootstrap); - - ShenandoahHeap* heap = ShenandoahHeap::heap(); - if (heap->mode()->is_generational()) { - heap->cancel_mixed_collections(); - } -} - diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp index 24b4b81fd424e..098aad6fa6ce0 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp @@ -42,8 +42,6 @@ class ShenandoahGlobalGeneration : public ShenandoahGeneration { virtual size_t used() const override; virtual size_t available() const override; - virtual void prepare_gc(bool do_old_gc_bootstrap) override; - virtual void set_concurrent_mark_in_progress(bool in_progress) override; bool contains(ShenandoahHeapRegion* region) const override; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index ffe1fc89003a7..9202c98333337 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -981,9 +981,26 @@ void ShenandoahHeap::retire_plab(PLAB* plab) { } } -void ShenandoahHeap::cancel_mixed_collections() { +void ShenandoahHeap::cancel_old_gc() { + shenandoah_assert_safepoint(); assert(_old_generation != NULL, "Should only have mixed collections in generation mode."); + log_info(gc)("Terminating old gc cycle."); + + // Stop marking + old_generation()->cancel_marking(); + // Stop coalescing undead objects + set_concurrent_prep_for_mixed_evacuation_in_progress(false); + // Stop tracking old regions old_heuristics()->abandon_collection_candidates(); + // Remove old generation access to young generation mark queues + young_generation()->set_old_gen_task_queues(nullptr); +} + +bool ShenandoahHeap::is_old_gc_active() { + return is_concurrent_old_mark_in_progress() + || is_concurrent_prep_for_mixed_evacuation_in_progress() + || old_heuristics()->unprocessed_old_or_hidden_collection_candidates() > 0 + || young_generation()->old_gen_task_queues() != nullptr; } void ShenandoahHeap::coalesce_and_fill_old_regions() { @@ -2111,18 +2128,10 @@ bool ShenandoahHeap::try_cancel_gc() { assert(prev == NOT_CANCELLED, "must be NOT_CANCELLED"); Thread* thread = Thread::current(); if (thread->is_Java_thread()) { - JavaThread* java_thread = JavaThread::cast(thread); - if (java_thread->thread_state() == _thread_in_Java) { - // ThreadBlockInVM requires thread state to be _thread_in_vm. If we are in Java, safely transition thread state. - ThreadInVMfromJava transition(java_thread); - // We need to provide a safepoint here. Otherwise we might spin forever if a SP is pending. - ThreadBlockInVM sp(JavaThread::cast(thread)); - SpinPause(); - } else { - // We need to provide a safepoint here. Otherwise we might spin forever if a SP is pending. - ThreadBlockInVM sp(JavaThread::cast(thread)); - SpinPause(); - } + // We need to provide a safepoint here, otherwise we might + // spin forever if a SP is pending. + ThreadBlockInVM sp(JavaThread::cast(thread)); + SpinPause(); } } } @@ -2810,8 +2819,8 @@ void ShenandoahHeap::flush_liveness_cache(uint worker_id) { } } -void ShenandoahHeap::purge_old_satb_buffers(bool abandon) { - ((ShenandoahOldGeneration*)_old_generation)->purge_satb_buffers(abandon); +void ShenandoahHeap::transfer_old_pointers_from_satb() { + ((ShenandoahOldGeneration*) _old_generation)->transfer_pointers_from_satb(); } template<> diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index 2ad657831af08..e87cc15723471 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -810,7 +810,8 @@ class ShenandoahHeap : public CollectedHeap { void clear_cards(HeapWord* start, HeapWord* end); void mark_card_as_dirty(void* location); void retire_plab(PLAB* plab); - void cancel_mixed_collections(); + void cancel_old_gc(); + bool is_old_gc_active(); void coalesce_and_fill_old_regions(); // ---------- Helper functions @@ -838,7 +839,7 @@ class ShenandoahHeap : public CollectedHeap { static inline void increase_object_age(oop obj, uint additional_age); - void purge_old_satb_buffers(bool abandon); + void transfer_old_pointers_from_satb(); private: void trash_cset_regions(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index 127f4037d0674..6ec2cffb8ecea 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -825,7 +825,7 @@ void ShenandoahHeapRegion::set_affiliation(ShenandoahRegionAffiliation new_affil { ShenandoahMarkingContext* const ctx = heap->complete_marking_context(); log_debug(gc)("Setting affiliation of Region " SIZE_FORMAT " from %s to %s, top: " PTR_FORMAT ", TAMS: " PTR_FORMAT - ", watermark: " PTR_FORMAT ", top_bitmap: " PTR_FORMAT "\n", + ", watermark: " PTR_FORMAT ", top_bitmap: " PTR_FORMAT, index(), affiliation_name(_affiliation), affiliation_name(new_affiliation), p2i(top()), p2i(ctx->top_at_mark_start(this)), p2i(_update_watermark), p2i(ctx->top_bitmap(this))); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp index 34940f83f0e9a..af8f502923915 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp @@ -300,8 +300,10 @@ inline void ShenandoahMark::mark_through_ref(T *p, ShenandoahObjToScanQueue* q, // Old mark, found a young pointer. // TODO: Rethink this: may be redundant with dirtying of cards identified during young-gen remembered set scanning // and by mutator write barriers. Assert - assert(heap->is_in_young(obj), "Expected young object."); - heap->mark_card_as_dirty(p); + if (heap->is_in(p)) { + assert(heap->is_in_young(obj), "Expected young object."); + heap->mark_card_as_dirty(p); + } } } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.inline.hpp index 2099726fa8ea6..90a6ca2345bf1 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.inline.hpp @@ -89,7 +89,7 @@ inline void ShenandoahMarkingContext::capture_top_at_mark_start(ShenandoahHeapRe "Region " SIZE_FORMAT ", bitmap should be clear while adjusting TAMS: " PTR_FORMAT " -> " PTR_FORMAT, idx, p2i(old_tams), p2i(new_tams)); - log_debug(gc)("Capturing TAMS for %s Region " SIZE_FORMAT ", was: %llx, now: %llx\n", + log_debug(gc)("Capturing TAMS for %s Region " SIZE_FORMAT ", was: %llx, now: %llx", affiliation_name(r->affiliation()), idx, (unsigned long long) old_tams, (unsigned long long) new_tams); if ((old_tams == r->bottom()) && (new_tams > old_tams)) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp index ac12ff3d215d4..de70a99c1b8c2 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp @@ -146,7 +146,7 @@ bool ShenandoahOldGC::collect(GCCause::Cause cause) { } } - if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_mark)) { + if (heap->cancelled_gc()) { return false; } @@ -183,29 +183,32 @@ bool ShenandoahOldGC::collect(GCCause::Cause cause) { heap->set_concurrent_prep_for_mixed_evacuation_in_progress(true); } + + assert(!heap->is_concurrent_strong_root_in_progress(), "No evacuations during old gc."); + + // We must execute this vm operation if we completed final mark. We cannot + // return from here with weak roots in progress. This is not a valid gc state + // for any young collections (or allocation failures) that interrupt the old + // collection. + vmop_entry_final_roots(false); + // Coalesce and fill objects _after_ weak root processing and class unloading. // Weak root and reference processing makes assertions about unmarked referents // that will fail if they've been overwritten with filler objects. There is also // a case in the LRB that permits access to from-space objects for the purpose // of class unloading that is unlikely to function correctly if the object has // been filled. - _allow_preemption.set(); - if (check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_evac)) { + if (heap->cancelled_gc()) { return false; } - assert(!heap->is_concurrent_strong_root_in_progress(), "No evacuations during old gc."); - - vmop_entry_final_roots(false); - if (heap->is_concurrent_prep_for_mixed_evacuation_in_progress()) { if (!entry_coalesce_and_fill()) { - // If old-gen degenerates instead of resuming, we'll just start up an out-of-cycle degenerated GC. - // This should be a rare event. Normally, we'll resume the coalesce-and-fill effort after the - // preempting young-gen GC finishes. - check_cancellation_and_abort(ShenandoahDegenPoint::_degenerated_outside_cycle); + // If an allocation failure occurs during coalescing, we will run a degenerated + // cycle for the young generation. This should be a rare event. Normally, we'll + // resume the coalesce-and-fill effort after the preempting young-gen GC finishes. return false; } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp index 1640b2ae6d83c..f9f6800b66e82 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp @@ -81,12 +81,10 @@ class ShenandoahProcessOldSATB : public SATBBufferClosure { for (size_t i = 0; i < size; ++i) { oop *p = (oop *) &buffer[i]; ShenandoahHeapRegion* region = _heap->heap_region_containing(*p); - if (region->is_old()) { - if (!region->is_trash()) { + if (region->is_old() && region->is_active()) { ShenandoahMark::mark_through_ref(p, _queue, NULL, _mark_context, false); - } else { - ++_trashed_oops; - } + } else { + ++_trashed_oops; } } } @@ -158,20 +156,24 @@ bool ShenandoahOldGeneration::is_concurrent_mark_in_progress() { return ShenandoahHeap::heap()->is_concurrent_old_mark_in_progress(); } -void ShenandoahOldGeneration::purge_satb_buffers(bool abandon) { +void ShenandoahOldGeneration::cancel_marking() { + if (is_concurrent_mark_in_progress()) { + ShenandoahBarrierSet::satb_mark_queue_set().abandon_partial_marking(); + } + + ShenandoahGeneration::cancel_marking(); +} + +void ShenandoahOldGeneration::transfer_pointers_from_satb() { ShenandoahHeap *heap = ShenandoahHeap::heap(); shenandoah_assert_safepoint(); assert(heap->is_concurrent_old_mark_in_progress(), "Only necessary during old marking."); - if (abandon) { - ShenandoahBarrierSet::satb_mark_queue_set().abandon_partial_marking(); - } else { - uint nworkers = heap->workers()->active_workers(); - StrongRootsScope scope(nworkers); + uint nworkers = heap->workers()->active_workers(); + StrongRootsScope scope(nworkers); - ShenandoahPurgeSATBTask purge_satb_task(task_queues()); - heap->workers()->run_task(&purge_satb_task); - } + ShenandoahPurgeSATBTask purge_satb_task(task_queues()); + heap->workers()->run_task(&purge_satb_task); } bool ShenandoahOldGeneration::contains(oop obj) const { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp index 1e61d14641d0c..49dae3e75c156 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp @@ -46,6 +46,8 @@ class ShenandoahOldGeneration : public ShenandoahGeneration { void set_concurrent_mark_in_progress(bool in_progress) override; + virtual void cancel_marking() override; + bool prepare_regions_and_collection_set(bool concurrent) override; virtual ShenandoahHeuristics* initialize_heuristics(ShenandoahMode* gc_mode) override; @@ -68,7 +70,7 @@ class ShenandoahOldGeneration : public ShenandoahGeneration { // Alternatively, we could inspect the state of the heap and the age of the // object at the barrier, but we reject this approach because it is likely // the performance impact would be too severe. - void purge_satb_buffers(bool abandon); + void transfer_pointers_from_satb(); bool is_concurrent_mark_in_progress() override; }; From 576b7dc8a134a690701682c052406819eebf81f9 Mon Sep 17 00:00:00 2001 From: Martin Doerr Date: Thu, 17 Mar 2022 16:31:23 +0000 Subject: [PATCH 116/254] Generational ShenandoahGC on PPC64 Reviewed-by: kdnilsen --- .../c1/shenandoahBarrierSetC1_ppc.cpp | 12 ++- .../shenandoahBarrierSetAssembler_ppc.cpp | 78 +++++++++++++++++-- .../shenandoahBarrierSetAssembler_ppc.hpp | 14 +++- 3 files changed, 97 insertions(+), 7 deletions(-) diff --git a/src/hotspot/cpu/ppc/gc/shenandoah/c1/shenandoahBarrierSetC1_ppc.cpp b/src/hotspot/cpu/ppc/gc/shenandoah/c1/shenandoahBarrierSetC1_ppc.cpp index fc06e1b71e0b8..02d219973a953 100644 --- a/src/hotspot/cpu/ppc/gc/shenandoah/c1/shenandoahBarrierSetC1_ppc.cpp +++ b/src/hotspot/cpu/ppc/gc/shenandoah/c1/shenandoahBarrierSetC1_ppc.cpp @@ -104,6 +104,8 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess &access, LI __ append(new LIR_OpShenandoahCompareAndSwap(addr, cmp_value.result(), new_value.result(), t1, t2, result)); + post_barrier(access, access.resolved_addr(), new_value.result()); + if (support_IRIW_for_not_multiple_copy_atomic_cpu) { __ membar_acquire(); } else { @@ -114,7 +116,13 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess &access, LI } } - return BarrierSetC1::atomic_cmpxchg_at_resolved(access, cmp_value, new_value); + LIR_Opr result = BarrierSetC1::atomic_cmpxchg_at_resolved(access, cmp_value, new_value); + + if (access.is_oop()) { + post_barrier(access, access.resolved_addr(), new_value.result()); + } + + return result; } LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess &access, LIRItem &value) { @@ -150,6 +158,8 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_xchg_at_resolved(LIRAccess &access, LIRIt if (ShenandoahSATBBarrier) { pre_barrier(access.gen(), access.access_emit_info(), access.decorators(), LIR_OprFact::illegalOpr, result); } + + post_barrier(access, access.resolved_addr(), result); } if (support_IRIW_for_not_multiple_copy_atomic_cpu) { diff --git a/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp b/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp index 8c9955078ffad..d8dae6a039d0f 100644 --- a/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp +++ b/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp @@ -37,6 +37,7 @@ #include "gc/shenandoah/shenandoahRuntime.hpp" #include "gc/shenandoah/shenandoahThreadLocalData.hpp" #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" +#include "gc/shenandoah/mode/shenandoahMode.hpp" #include "interpreter/interpreter.hpp" #include "runtime/sharedRuntime.hpp" #include "runtime/thread.hpp" @@ -93,8 +94,6 @@ void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler *masm, void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler *masm, DecoratorSet decorators, BasicType type, Register src, Register dst, Register count, Register preserve1, Register preserve2) { - __ block_comment("arraycopy_prologue (shenandoahgc) {"); - Register R11_tmp = R11_scratch1; assert_different_registers(src, dst, count, R11_tmp, noreg); @@ -117,6 +116,7 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler *masm, Dec return; } + __ block_comment("arraycopy_prologue (shenandoahgc) {"); Label skip_prologue; // Fast path: Array is of length zero. @@ -133,7 +133,7 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler *masm, Dec // for the garbage collector. const int required_states = ShenandoahSATBBarrier && dest_uninitialized ? ShenandoahHeap::HAS_FORWARDED - : ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING; + : ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING; __ andi_(R11_tmp, R11_tmp, required_states); __ beq(CCR0, skip_prologue); @@ -190,6 +190,16 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler *masm, Dec __ block_comment("} arraycopy_prologue (shenandoahgc)"); } +void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Register dst, Register count, + Register preserve) { + if (is_reference_type(type)) { + __ block_comment("arraycopy_epilogue (shenandoahgc) {"); + gen_write_ref_array_post_barrier(masm, decorators, dst, count, preserve); + __ block_comment("} arraycopy_epilogue (shenandoahgc)"); + } +} + // The to-be-enqueued value can either be determined // - dynamically by passing the reference's address information (load mode) or // - statically by passing a register the value is stored in (preloaded mode) @@ -223,7 +233,7 @@ void ShenandoahBarrierSetAssembler::satb_write_barrier_impl(MacroAssembler *masm // Check whether marking is active. __ lbz(tmp1, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread); - __ andi_(tmp1, tmp1, ShenandoahHeap::MARKING); + __ andi_(tmp1, tmp1, ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING); __ beq(CCR0, skip_barrier); /* ==== Determine the reference's previous value ==== */ @@ -589,6 +599,27 @@ void ShenandoahBarrierSetAssembler::load_at( } } +void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register base, RegisterOrConstant ind_or_offs, Register tmp) { + if (!ShenandoahHeap::heap()->mode()->is_generational()) { + return; + } + + ShenandoahBarrierSet* ctbs = ShenandoahBarrierSet::barrier_set(); + CardTable* ct = ctbs->card_table(); + assert_different_registers(base, tmp, R0); + + if (ind_or_offs.is_constant()) { + __ add_const_optimized(base, base, ind_or_offs.as_constant(), tmp); + } else { + __ add(base, ind_or_offs.as_register(), base); + } + + __ load_const_optimized(tmp, (address)ct->byte_map_base(), R0); + __ srdi(base, base, CardTable::card_shift()); + __ li(R0, CardTable::dirty_card_val()); + __ stbx(R0, tmp, base); +} + // base: Base register of the reference's address. // ind_or_offs: Index or offset of the reference's address. // val: To-be-stored value/reference's new value. @@ -611,6 +642,11 @@ void ShenandoahBarrierSetAssembler::store_at(MacroAssembler *masm, DecoratorSet val, tmp1, tmp2, tmp3, preservation_level); + + // No need for post barrier if storing NULL + if (is_reference_type(type) && val != noreg) { + store_check(masm, base, ind_or_offs, tmp1); + } } void ShenandoahBarrierSetAssembler::try_resolve_jobject_in_native(MacroAssembler *masm, @@ -760,6 +796,38 @@ void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler *masm, Register b __ block_comment("} cmpxchg_oop (shenandoahgc)"); } +void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, + Register addr, Register count, Register preserve) { + if (!ShenandoahHeap::heap()->mode()->is_generational()) { + return; + } + + ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set(); + CardTable* ct = bs->card_table(); + assert_different_registers(addr, count, R0); + + Label Lskip_loop, Lstore_loop; + + __ sldi_(count, count, LogBytesPerHeapOop); + __ beq(CCR0, Lskip_loop); // zero length + __ addi(count, count, -BytesPerHeapOop); + __ add(count, addr, count); + // Use two shifts to clear out those low order two bits! (Cannot opt. into 1.) + __ srdi(addr, addr, CardTable::card_shift()); + __ srdi(count, count, CardTable::card_shift()); + __ subf(count, addr, count); + __ add_const_optimized(addr, addr, (address)ct->byte_map_base(), R0); + __ addi(count, count, 1); + __ li(R0, 0); + __ mtctr(count); + // Byte store loop + __ bind(Lstore_loop); + __ stb(R0, 0, addr); + __ addi(addr, addr, 1); + __ bdnz(Lstore_loop); + __ bind(Lskip_loop); +} + #undef __ #ifdef COMPILER1 @@ -895,7 +963,7 @@ void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAss // another check is required as a safepoint might have been reached in the meantime (JDK-8140588). __ lbz(R12_tmp2, in_bytes(ShenandoahThreadLocalData::gc_state_offset()), R16_thread); - __ andi_(R12_tmp2, R12_tmp2, ShenandoahHeap::MARKING); + __ andi_(R12_tmp2, R12_tmp2, ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING); __ beq(CCR0, skip_barrier); /* ==== Add previous value directly to thread-local SATB mark queue ==== */ diff --git a/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.hpp b/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.hpp index cf55f505b2207..f8690e45eadeb 100644 --- a/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.hpp +++ b/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.hpp @@ -51,6 +51,10 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler { Register tmp1, Register tmp2, MacroAssembler::PreservationLevel preservation_level); + void store_check(MacroAssembler* masm, + Register base, RegisterOrConstant ind_or_offs, + Register tmp); + void load_reference_barrier_impl(MacroAssembler* masm, DecoratorSet decorators, Register base, RegisterOrConstant ind_or_offs, Register dst, @@ -60,6 +64,10 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler { /* ==== Helper methods for barrier implementations ==== */ void resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp); + void gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, + Register addr, Register count, + Register preserve); + public: /* ==== C1 stubs ==== */ @@ -99,7 +107,11 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler { /* ==== Access api ==== */ virtual void arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, - Register src, Register dst, Register count, Register preserve1, Register preserve2); + Register src, Register dst, Register count, + Register preserve1, Register preserve2); + virtual void arraycopy_epilogue(MacroAssembler* masm, DecoratorSet decorators, BasicType type, + Register dst, Register count, + Register preserve); virtual void store_at(MacroAssembler* masm, DecoratorSet decorators, BasicType type, Register base, RegisterOrConstant ind_or_offs, Register val, From 71fe3e547c9da1ca695fb405f7b2b034548f41cc Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Mon, 21 Mar 2022 20:31:49 +0000 Subject: [PATCH 117/254] Do not abort coalesce and fill during fullgc Reviewed-by: wkemper --- .../share/gc/shenandoah/shenandoahFullGC.cpp | 3 +- .../gc/shenandoah/shenandoahHeapRegion.cpp | 41 +++++++++++++++++++ .../gc/shenandoah/shenandoahHeapRegion.hpp | 3 ++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index 25349704d62ad..eb12a9fb83501 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -159,6 +159,7 @@ void ShenandoahFullGC::entry_full(GCCause::Cause cause) { } void ShenandoahFullGC::op_full(GCCause::Cause cause) { + ShenandoahHeap* const heap = ShenandoahHeap::heap(); ShenandoahMetricsSnapshot metrics; metrics.snap_before(); @@ -1135,7 +1136,7 @@ class ShenandoahAdjustPointersTask : public WorkerTask { // reference to reclaimed memory. Remembered set scanning will crash if it attempts // to iterate the oops in these objects. r->begin_preemptible_coalesce_and_fill(); - r->oop_fill_and_coalesce(); + r->oop_fill_and_coalesce_wo_cancel(); } r = _regions.next(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index 6ec2cffb8ecea..0ecd4b6bc7502 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -423,6 +423,47 @@ void ShenandoahHeapRegion::print_on(outputStream* st) const { st->cr(); } +// oop_iterate without closure and without cancellation. always return true. +bool ShenandoahHeapRegion::oop_fill_and_coalesce_wo_cancel() { + HeapWord* obj_addr = resume_coalesce_and_fill(); + + assert(!is_humongous(), "No need to fill or coalesce humongous regions"); + if (!is_active()) { + end_preemptible_coalesce_and_fill(); + return true; + } + + ShenandoahHeap* heap = ShenandoahHeap::heap(); + ShenandoahMarkingContext* marking_context = heap->marking_context(); + // All objects above TAMS are considered live even though their mark bits will not be set. Note that young- + // gen evacuations that interrupt a long-running old-gen concurrent mark may promote objects into old-gen + // while the old-gen concurrent marking is ongoing. These newly promoted objects will reside above TAMS + // and will be treated as live during the current old-gen marking pass, even though they will not be + // explicitly marked. + HeapWord* t = marking_context->top_at_mark_start(this); + + // Expect marking to be completed before these threads invoke this service. + assert(heap->active_generation()->is_mark_complete(), "sanity"); + while (obj_addr < t) { + oop obj = cast_to_oop(obj_addr); + if (marking_context->is_marked(obj)) { + assert(obj->klass() != NULL, "klass should not be NULL"); + obj_addr += obj->size(); + } else { + // Object is not marked. Coalesce and fill dead object with dead neighbors. + HeapWord* next_marked_obj = marking_context->get_next_marked_addr(obj_addr, t); + assert(next_marked_obj <= t, "next marked object cannot exceed top"); + size_t fill_size = next_marked_obj - obj_addr; + ShenandoahHeap::fill_with_object(obj_addr, fill_size); + heap->card_scan()->coalesce_objects(obj_addr, fill_size); + obj_addr = next_marked_obj; + } + } + // Mark that this region has been coalesced and filled + end_preemptible_coalesce_and_fill(); + return true; +} + // oop_iterate without closure, return true if completed without cancellation bool ShenandoahHeapRegion::oop_fill_and_coalesce() { HeapWord* obj_addr = resume_coalesce_and_fill(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp index 8e7ac4bec7d1c..336aff6f43983 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp @@ -393,6 +393,9 @@ class ShenandoahHeapRegion { // region is completely coalesced and filled. Returns false if cancelled before task is complete. bool oop_fill_and_coalesce(); + // Like oop_fill_and_coalesce(), but without honoring cancellation requests. + bool oop_fill_and_coalesce_wo_cancel(); + // During global collections, this service iterates through an old-gen heap region that is not part of collection // set to fill and register ranges of dead memory. Note that live objects were previously registered. Some dead objects // that are subsumed into coalesced ranges of dead memory need to be "unregistered". From ca6fa69e238c747d1f83514d52aeb39d923bce51 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Mon, 21 Mar 2022 20:45:51 +0000 Subject: [PATCH 118/254] Global mark is not old mark Reviewed-by: kdnilsen --- src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp index a7299fab1864b..fd576c224e2ce 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp @@ -67,7 +67,6 @@ void ShenandoahGlobalGeneration::set_concurrent_mark_in_progress(bool in_progres } heap->set_concurrent_young_mark_in_progress(in_progress); - heap->set_concurrent_old_mark_in_progress(in_progress); } bool ShenandoahGlobalGeneration::contains(ShenandoahHeapRegion* region) const { From 5fd97640bc0f21d1f5512a84f1548add89486ea3 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Thu, 24 Mar 2022 23:33:36 +0000 Subject: [PATCH 119/254] Balance evacuation Reviewed-by: wkemper --- .../share/gc/shenandoah/shenandoahHeap.cpp | 13 +++++++ .../gc/shenandoah/shenandoah_globals.hpp | 37 +++++++++++++++++-- 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 9202c98333337..9eb4fa42f0ee6 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -484,6 +484,8 @@ void ShenandoahHeap::initialize_heuristics() { if (mode()->is_generational()) { _young_generation->initialize_heuristics(_gc_mode); _old_generation->initialize_heuristics(_gc_mode); + + ShenandoahEvacWaste = ShenandoahGenerationalEvacWaste; } } @@ -847,6 +849,12 @@ HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) // Figure out size of new GCLAB, looking back at heuristics. Expand aggressively. size_t new_size = ShenandoahThreadLocalData::gclab_size(thread) * 2; + + // Limit growth of GCLABs to ShenandoahMaxEvacLABRatio * the minimum size. This enables more equitable distribution of + // available evacuation buidget between the many threads that are coordinating in the evacuation effort. + if (ShenandoahMaxEvacLABRatio > 0) { + new_size = MIN2(new_size, PLAB::min_size() * ShenandoahMaxEvacLABRatio); + } new_size = MIN2(new_size, PLAB::max_size()); new_size = MAX2(new_size, PLAB::min_size()); @@ -897,6 +905,11 @@ HeapWord* ShenandoahHeap::allocate_from_plab_slow(Thread* thread, size_t size, b // Figure out size of new PLAB, looking back at heuristics. Expand aggressively. size_t new_size = ShenandoahThreadLocalData::plab_size(thread) * 2; + // Limit growth of PLABs to ShenandoahMaxEvacLABRatio * the minimum size. This enables more equitable distribution of + // available evacuation buidget between the many threads that are coordinating in the evacuation effort. + if (ShenandoahMaxEvacLABRatio > 0) { + new_size = MIN2(new_size, PLAB::min_size() * ShenandoahMaxEvacLABRatio); + } new_size = MIN2(new_size, PLAB::max_size()); new_size = MAX2(new_size, PLAB::min_size()); diff --git a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp index ad94cf745cc99..e783a2ff32518 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp @@ -281,11 +281,42 @@ "Larger values make evacuations more resilient against " \ "evacuation conflicts, at expense of evacuating less on each " \ "GC cycle. Smaller values increase the risk of evacuation " \ - "failures, which will trigger stop-the-world Full GC passes. " \ - "A minimum value of 1.6 is recommended for Generational " \ - "mode of Shenandoah.") \ + "failures, which will trigger stop-the-world Full GC passes.") \ range(1.0,100.0) \ \ + product(double, ShenandoahGenerationalEvacWaste, 2.0, EXPERIMENTAL, \ + "For generational mode, how much waste evacuations produce " \ + "within the reserved space. Larger values make evacuations " \ + "more resilient against evacuation conflicts, at expense of " \ + "evacuating less on each GC cycle. Smaller values increase " \ + "the risk of evacuation failures, which will trigger " \ + "stop-the-world Full GC passes. The default value for " \ + "generational mode is 2.0. The reason for the higher default " \ + "value in generational mode is because generational mode " \ + "enforces the evacuation budget, triggering degenerated GC " \ + "which upgrades to full GC whenever the budget is exceeded.") \ + range(1.0,100.0) \ + \ + product(uintx, ShenandoahMaxEvacLABRatio, 16, EXPERIMENTAL, \ + "Potentially, each running thread maintains a PLAB for " \ + "evacuating objects into old-gen memory and a GCLAB for " \ + "evacuating objects into young-gen memory. Each time a thread " \ + "exhausts its PLAB or GCLAB, a new local buffer is allocated. " \ + "By default, the new buffer is twice the size of the previous " \ + "buffer. The sizes are reset to the minimum at the start of " \ + "each GC pass. This parameter limits the growth of evacuation " \ + "buffer sizes to its value multiplied by the minimum buffer " \ + "size. A higher value allows evacuation allocations to be more " \ + "efficient because less synchronization is required by " \ + "individual threads. However, a larger value increases the " \ + "likelihood of evacuation failures, leading to long " \ + "stop-the-world pauses. This is because a large value " \ + "allows individual threads to consume large percentages of " \ + "the total evacuation budget without necessarily effectively " \ + "filling their local evcauation buffers with evacuated " \ + "objects. A value of zero means no maximum size is enforced.") \ + range(0, 1024) \ + \ product(bool, ShenandoahEvacReserveOverflow, true, EXPERIMENTAL, \ "Allow evacuations to overflow the reserved space. Enabling it " \ "will make evacuations more resilient when evacuation " \ From 54f1e49604b2c4e91b4f484de24cefed4409ecf2 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Wed, 6 Apr 2022 18:36:39 +0000 Subject: [PATCH 120/254] Relax enforcement of evacuation budget Reviewed-by: kdnilsen --- .../heuristics/shenandoahHeuristics.cpp | 13 +-------- .../gc/shenandoah/shenandoahConcurrentGC.cpp | 2 -- .../gc/shenandoah/shenandoahDegeneratedGC.cpp | 2 -- .../share/gc/shenandoah/shenandoahFreeSet.cpp | 11 +------- .../share/gc/shenandoah/shenandoahFullGC.cpp | 2 -- .../gc/shenandoah/shenandoahGeneration.cpp | 1 + .../share/gc/shenandoah/shenandoahHeap.cpp | 28 +++---------------- .../share/gc/shenandoah/shenandoahHeap.hpp | 19 ++----------- .../gc/shenandoah/shenandoahHeap.inline.hpp | 13 --------- .../gc/shenandoah/shenandoahOldGeneration.cpp | 5 ++-- .../gc/shenandoah/shenandoahVerifier.cpp | 11 +------- 11 files changed, 14 insertions(+), 93 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp index 1dfb70a68ba4b..b534976a44c9c 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp @@ -134,7 +134,7 @@ bool ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec candidates[cand_idx]._region = region; if (heap->mode()->is_generational() && (region->age() >= InitialTenuringThreshold)) { // Bias selection of regions that have reached tenure age - for (int i = region->age() - InitialTenuringThreshold; i >= 0; i--) { + for (uint j = region->age() - InitialTenuringThreshold; j > 0; j--) { garbage = (garbage + ShenandoahTenuredRegionUsageBias) * ShenandoahTenuredRegionUsageBias; } } @@ -219,7 +219,6 @@ bool ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec size_t young_available = (free_affiliated_regions + young_generation->free_unaffiliated_regions()) * region_size_bytes; size_t regions_available_to_loan = 0; - size_t preapproved_evac_reserve_loan = 0; if (heap->mode()->is_generational()) { // Now that we've primed the collection set, we can figure out how much memory to reserve for evacuation @@ -239,8 +238,6 @@ bool ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec // young_evacuation_reserve -= heap->get_old_evac_reserve(); - size_t old_region_borrow_count = 0; - // Though we cannot know the evacuation_supplement until after we have computed the collection set, we do // know that every young-gen region added to the collection set will have a net positive impact on available // memory within young-gen, since each contributes a positive amount of garbage to available. Thus, even @@ -248,7 +245,6 @@ bool ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec // exceed young_available if there are empty regions available within old-gen to hold the results of evacuation. ShenandoahGeneration* old_generation = heap->old_generation(); - ShenandoahYoungGeneration* young_generation = heap->young_generation(); // Not all of what is currently available within young-gen can be reserved to hold the results of young-gen // evacuation. This is because memory available within any heap region that is placed into the collection set @@ -274,7 +270,6 @@ bool ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec } else { // No need to scale back young_evacuation_reserve. } - preapproved_evac_reserve_loan = loaned_region_request * region_size_bytes; } else { // No need scale back young_evacuation_reserve and no need to borrow from old-gen. We may even have some // available_young_regions to support allocation pacing. @@ -286,16 +281,10 @@ bool ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec } heap->set_young_evac_reserve(young_evacuation_reserve); - heap->reset_young_evac_expended(); // Add young-gen regions into the collection set. This is a virtual call, implemented differently by each // of the heuristics subclasses. choose_collection_set_from_regiondata(collection_set, candidates, cand_idx, immediate_garbage + free); - size_t young_evacuated_bytes = collection_set->get_young_bytes_reserved_for_evacuation();; - if (young_evacuated_bytes * ShenandoahEvacWaste < young_evacuation_reserve) { - young_evacuation_reserve = (size_t) (young_evacuated_bytes * ShenandoahEvacWaste); - heap->set_young_evac_reserve((size_t) young_evacuation_reserve); - } // Now compute the evacuation supplement, which is extra memory borrowed from old-gen that can be allocated // by mutators while GC is working on evacuation and update-refs. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index 6c607b1ad5526..cc3704312233e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -228,7 +228,6 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) { young_gen->unadjust_available(); old_gen->unadjust_available(); - young_gen->increase_used(heap->get_young_evac_expended()); // No need to old_gen->increase_used(). That was done when plabs were allocated, accounting for both old evacs and promotions. young_available = young_gen->adjusted_available(); @@ -236,7 +235,6 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) { heap->set_alloc_supplement_reserve(0); heap->set_young_evac_reserve(0); - heap->reset_young_evac_expended(); heap->set_old_evac_reserve(0); heap->reset_old_evac_expended(); heap->set_promotion_reserve(0); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp index 9fafda229cc43..ec44434148219 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp @@ -280,12 +280,10 @@ void ShenandoahDegenGC::op_degenerated() { heap->young_generation()->unadjust_available(); heap->old_generation()->unadjust_available(); - heap->young_generation()->increase_used(heap->get_young_evac_expended()); // No need to old_gen->increase_used(). That was done when plabs were allocated, accounting for both old evacs and promotions. heap->set_alloc_supplement_reserve(0); heap->set_young_evac_reserve(0); - heap->reset_young_evac_expended(); heap->set_old_evac_reserve(0); heap->reset_old_evac_expended(); heap->set_promotion_reserve(0); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index be6d9882c21e4..312d87a38a8a1 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -293,18 +293,9 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah r->set_update_watermark(r->top()); if (r->affiliation() == ShenandoahRegionAffiliation::YOUNG_GENERATION) { - // This is either a GCLAB or it is a shared evacuation allocation. In either case, we expend young evac. - // At end of update refs, we'll add expended young evac into young_gen->used. We hide this usage - // from current accounting because memory reserved for evacuation is not part of adjusted capacity. - if (_heap->mode()->is_generational()) { - _heap->expend_young_evac(size * HeapWordSize); - } else { - // If we are not in generational mode, we still need to count this allocation as used memory. - _heap->young_generation()->increase_used(size * HeapWordSize); - } + _heap->young_generation()->increase_used(size * HeapWordSize); } else { assert(r->affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION, "GC Alloc was not YOUNG so must be OLD"); - assert(req.type() != ShenandoahAllocRequest::_alloc_gclab, "old-gen allocations use PLAB or shared allocation"); _heap->old_generation()->increase_used(size * HeapWordSize); // for plabs, we'll sort the difference between evac and promotion usage when we retire the plab diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index eb12a9fb83501..eca957f1f1805 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -185,12 +185,10 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) { // There will be no concurrent allocations during full GC so reset these coordination variables. heap->young_generation()->unadjust_available(); heap->old_generation()->unadjust_available(); - heap->young_generation()->increase_used(heap->get_young_evac_expended()); // No need to old_gen->increase_used(). That was done when plabs were allocated, accounting for both old evacs and promotions. heap->set_alloc_supplement_reserve(0); heap->set_young_evac_reserve(0); - heap->reset_young_evac_expended(); heap->set_old_evac_reserve(0); heap->reset_old_evac_expended(); heap->set_promotion_reserve(0); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index f6f365410945e..51f95d828992b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -396,6 +396,7 @@ ShenandoahMarkingContext* ShenandoahGeneration::complete_marking_context() { } void ShenandoahGeneration::cancel_marking() { + log_info(gc)("Cancel marking: %s", name()); if (is_concurrent_mark_in_progress()) { set_mark_incomplete(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 9eb4fa42f0ee6..61bb1e2d3e441 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -515,7 +515,6 @@ ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) : _old_evac_reserve(0), _old_evac_expended(0), _young_evac_reserve(0), - _young_evac_expended(0), _captured_old_usage(0), _previous_promotion(0), _cancel_requested_time(0), @@ -1167,31 +1166,12 @@ HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req ShenandoahHeapLocker locker(lock()); if (mode()->is_generational()) { if (req.affiliation() == YOUNG_GENERATION) { - if (req.type() == ShenandoahAllocRequest::_alloc_gclab) { - if (requested_bytes + get_young_evac_expended() > get_young_evac_reserve()) { - // This should only happen if evacuation waste is too low. Rejecting one thread's request for GCLAB does not - // necessarily result in failure of the evacuation effort. A different thread may be able to copy from-space object. - - // TODO: Should we really fail here in the case that there is sufficient memory to allow us to allocate a gclab - // beyond the young_evac_reserve? Seems it would be better to take away from mutator allocation budget if this - // prevents fall-back to full GC in order to recover from failed evacuation. + if (req.is_mutator_alloc()) { + if (requested_bytes >= young_generation()->adjusted_available()) { + // We know this is not a GCLAB. This must be a TLAB or a shared allocation. Reject the allocation request if + // exceeds established capacity limits. return nullptr; } - // else, there is sufficient memory to allocate this GCLAB so do nothing here. - } else if (req.is_gc_alloc()) { - // This is a shared alloc for purposes of evacuation. - if (requested_bytes + get_young_evac_expended() > get_young_evac_reserve()) { - // TODO: Should we really fail here in the case that there is sufficient memory to allow us to allocate a gclab - // beyond the young_evac_reserve? Seems it would be better to take away from mutator allocation budget if this - // prevents fall-back to full GC in order to recover from failed evacuation. - return nullptr; - } else { - // There is sufficient memory to allocate this shared evacuation object. - } - } else if (requested_bytes >= young_generation()->adjusted_available()) { - // We know this is not a GCLAB. This must be a TLAB or a shared allocation. Reject the allocation request if - // exceeds established capacity limits. - return nullptr; } } else { // reg.affiliation() == OLD_GENERATION assert(req.type() != ShenandoahAllocRequest::_alloc_gclab, "GCLAB pertains only to young-gen memory"); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index e87cc15723471..8ac2bfffd3896 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -346,29 +346,21 @@ class ShenandoahHeap : public CollectedHeap { // two values are constant throughout each GC phases, we introduce a new service into ShenandoahGeneration. This service // provides adjusted_available() based on an adjusted capacity. At the start of evacuation, we adjust young capacity by // adding the amount to be borrowed from old-gen and subtracting the _young_evac_reserve, we adjust old capacity by - // subtracting the amount to be loaned to young-gen. At the end of update-refs, we unadjust the capacity of each generation, - // and add _young_evac_expended to young-gen used. + // subtracting the amount to be loaned to young-gen. // // We always use adjusted capacities to determine permission to allocate within young and to promote into old. Note // that adjusted capacities equal traditional capacities except during evacuation and update refs. // - // During evacuation, we assure that _young_evac_expended does not exceed _young_evac_reserve and that _old_evac_expended - // does not exceed _old_evac_reserve. GCLAB allocations do not immediately affect used within the young generation - // since the adjusted capacity already accounts for the entire evacuation reserve. Each GCLAB allocations increments - // _young_evac_expended rather than incrementing the affiliated generation's used value. + // During evacuation, we assure that _old_evac_expended does not exceed _old_evac_reserve. // // At the end of update references, we perform the following bookkeeping activities: // // 1. Unadjust the capacity within young-gen and old-gen to undo the effects of borrowing memory from old-gen. Note that // the entirety of the collection set is now available, so allocation capacity naturally increase at this time. - // 2. Increase young_gen->used() by _young_evac_expended. This represents memory consumed by evacutions from young-gen. - // 3. Clear (reset to zero) _alloc_supplement_reserve, _young_evac_reserve, _old_evac_reserve, and _promotion_reserve + // 2. Clear (reset to zero) _alloc_supplement_reserve, _young_evac_reserve, _old_evac_reserve, and _promotion_reserve // // _young_evac_reserve and _old_evac_reserve are only non-zero during evacuation and update-references. // - // Allocation of young GCLABs assures that _young_evac_expended + request-size < _young_evac_reserved. If the allocation - // is authorized, increment _young_evac_expended by request size. This allocation ignores young_gen->available(). - // // Allocation of old GCLABs assures that _old_evac_expended + request-size < _old_evac_reserved. If the allocation // is authorized, increment _old_evac_expended by request size. This allocation ignores old_gen->available(). // @@ -386,7 +378,6 @@ class ShenandoahHeap : public CollectedHeap { size_t _old_evac_expended; // Bytes of old-gen memory expended on old-gen evacuations size_t _young_evac_reserve; // Bytes reserved within young-gen to hold evacuated objects from young-gen collection set - size_t _young_evac_expended; // Bytes old-gen memory has been expended on young-gen evacuations size_t _captured_old_usage; // What was old usage (bytes) when last captured? @@ -457,10 +448,6 @@ class ShenandoahHeap : public CollectedHeap { inline size_t set_young_evac_reserve(size_t new_val); inline size_t get_young_evac_reserve() const; - inline void reset_young_evac_expended(); - inline size_t expend_young_evac(size_t increment); - inline size_t get_young_evac_expended() const; - // Returns previous value. This is a signed value because it is the amount borrowed minus the amount reserved for // young-gen evacuation. In case we cannot borrow much, this value might be negative. inline intptr_t set_alloc_supplement_reserve(intptr_t new_val); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp index f47be76d23fc8..b8398d168cddf 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp @@ -646,19 +646,6 @@ inline size_t ShenandoahHeap::get_young_evac_reserve() const { return _young_evac_reserve; } -inline void ShenandoahHeap::reset_young_evac_expended() { - _young_evac_expended = 0; -} - -inline size_t ShenandoahHeap::expend_young_evac(size_t increment) { - _young_evac_expended += increment; - return _young_evac_expended; -} - -inline size_t ShenandoahHeap::get_young_evac_expended() const { - return _young_evac_expended; -} - inline intptr_t ShenandoahHeap::set_alloc_supplement_reserve(intptr_t new_val) { intptr_t orig = _alloc_supplement_reserve; _alloc_supplement_reserve = new_val; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp index f9f6800b66e82..6e1ff76568d83 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp @@ -158,6 +158,7 @@ bool ShenandoahOldGeneration::is_concurrent_mark_in_progress() { void ShenandoahOldGeneration::cancel_marking() { if (is_concurrent_mark_in_progress()) { + log_info(gc)("Abandon satb buffers."); ShenandoahBarrierSet::satb_mark_queue_set().abandon_partial_marking(); } @@ -165,10 +166,10 @@ void ShenandoahOldGeneration::cancel_marking() { } void ShenandoahOldGeneration::transfer_pointers_from_satb() { - ShenandoahHeap *heap = ShenandoahHeap::heap(); + ShenandoahHeap* heap = ShenandoahHeap::heap(); shenandoah_assert_safepoint(); assert(heap->is_concurrent_old_mark_in_progress(), "Only necessary during old marking."); - + log_info(gc)("Transfer satb buffers."); uint nworkers = heap->workers()->active_workers(); StrongRootsScope scope(nworkers); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp index 05493ccf71579..b63bde6c4fa50 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp @@ -388,16 +388,7 @@ class ShenandoahGenerationStatsClosure : public ShenandoahHeapRegionClosure { } static void validate_usage(const char* label, ShenandoahGeneration* generation, ShenandoahCalculateRegionStatsClosure& stats) { - size_t generation_used; - if (generation->generation_mode() == YOUNG) { - // young_evac_expended is "usually zero". If it is non-zero, this means we are doing evacuation or updating references - // and young-gen memory that holds the results of evacuation is being temporarily hidden from the usage accounting, - // so we add it back in here to make verification happy. - generation_used = generation->used() + ShenandoahHeap::heap()->get_young_evac_expended(); - } else { - generation_used = generation->used(); - } - + size_t generation_used = generation->used(); guarantee(stats.used() == generation_used, "%s: generation (%s) used size must be consistent: generation-used = " SIZE_FORMAT "%s, regions-used = " SIZE_FORMAT "%s", label, generation->name(), From 278ecdc56accfe26cf7fdab59d11a8e4926092a9 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Tue, 12 Apr 2022 16:25:25 +0000 Subject: [PATCH 121/254] Cool down regulator after request to start GC is accepted Reviewed-by: kdnilsen --- .../share/gc/shenandoah/shenandoahControlThread.cpp | 7 +++++++ .../share/gc/shenandoah/shenandoahControlThread.hpp | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index 48d4789522cf2..910eff2f00ee4 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -54,6 +54,7 @@ ShenandoahControlThread::ShenandoahControlThread() : _alloc_failure_waiters_lock(Mutex::safepoint - 1, "ShenandoahAllocFailureGC_lock", true), _gc_waiters_lock(Mutex::safepoint - 1, "ShenandoahRequestedGC_lock", true), _control_lock(Mutex::nosafepoint - 1, "ShenandoahControlGC_lock", true), + _regulator_lock(Mutex::nosafepoint - 1, "ShenandoahRegulatorGC_lock", true), _periodic_task(this), _requested_gc_cause(GCCause::_no_cause_specified), _requested_generation(GenerationMode::GLOBAL), @@ -741,6 +742,8 @@ bool ShenandoahControlThread::request_concurrent_gc(GenerationMode generation) { _requested_gc_cause = GCCause::_shenandoah_concurrent_gc; _requested_generation = generation; notify_control_thread(); + MonitorLocker ml(&_regulator_lock, Mutex::_no_safepoint_check_flag); + ml.wait(); return true; } @@ -751,6 +754,8 @@ bool ShenandoahControlThread::request_concurrent_gc(GenerationMode generation) { _preemption_requested.set(); ShenandoahHeap::heap()->cancel_gc(GCCause::_shenandoah_concurrent_gc); notify_control_thread(); + MonitorLocker ml(&_regulator_lock, Mutex::_no_safepoint_check_flag); + ml.wait(); return true; } @@ -929,5 +934,7 @@ void ShenandoahControlThread::set_gc_mode(ShenandoahControlThread::GCMode new_mo if (_mode != new_mode) { log_info(gc)("Transition from: %s to: %s", gc_mode_name(_mode), gc_mode_name(new_mode)); _mode = new_mode; + MonitorLocker ml(&_regulator_lock, Mutex::_no_safepoint_check_flag); + ml.notify_all(); } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp index 78180e3078a5e..e50b235b1077e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp @@ -59,10 +59,11 @@ class ShenandoahControlThread: public ConcurrentGCThread { private: // While we could have a single lock for these, it may risk unblocking // GC waiters when alloc failure GC cycle finishes. We want instead - // to make complete explicit cycle for for demanding customers. + // to make complete explicit cycle for demanding customers. Monitor _alloc_failure_waiters_lock; Monitor _gc_waiters_lock; Monitor _control_lock; + Monitor _regulator_lock; ShenandoahPeriodicTask _periodic_task; ShenandoahPeriodicPacerNotify _periodic_pacer_notify_task; From ca2cf6b871cc3ed250ad75c72a8a8330efd28f05 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Thu, 14 Apr 2022 21:17:37 +0000 Subject: [PATCH 122/254] Improved timing reports Reviewed-by: kdnilsen --- .../shenandoah/shenandoahCollectorPolicy.cpp | 16 ++++- .../shenandoah/shenandoahCollectorPolicy.hpp | 4 ++ .../gc/shenandoah/shenandoahConcurrentGC.cpp | 4 +- .../gc/shenandoah/shenandoahControlThread.cpp | 61 +++++++++++-------- .../gc/shenandoah/shenandoahControlThread.hpp | 2 + .../gc/shenandoah/shenandoahPhaseTimings.hpp | 6 +- 6 files changed, 62 insertions(+), 31 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.cpp index adb40e9d2af12..1cfd2a88a3f5a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.cpp @@ -31,6 +31,8 @@ ShenandoahCollectorPolicy::ShenandoahCollectorPolicy() : _success_concurrent_gcs(0), + _success_old_gcs(0), + _interrupted_old_gcs(0), _success_degenerated_gcs(0), _success_full_gcs(0), _alloc_failure_degenerated(0), @@ -83,6 +85,14 @@ void ShenandoahCollectorPolicy::record_success_concurrent() { _success_concurrent_gcs++; } +void ShenandoahCollectorPolicy::record_success_old() { + _success_old_gcs++; +} + +void ShenandoahCollectorPolicy::record_interrupted_old() { + _interrupted_old_gcs++; +} + void ShenandoahCollectorPolicy::record_success_degenerated() { _success_degenerated_gcs++; } @@ -114,11 +124,15 @@ void ShenandoahCollectorPolicy::print_gc_stats(outputStream* out) const { out->print_cr("to avoid Degenerated and Full GC cycles."); out->cr(); - out->print_cr(SIZE_FORMAT_W(5) " successful concurrent GCs", _success_concurrent_gcs); + out->print_cr(SIZE_FORMAT_W(5) " Successful Concurrent GCs", _success_concurrent_gcs); out->print_cr(" " SIZE_FORMAT_W(5) " invoked explicitly", _explicit_concurrent); out->print_cr(" " SIZE_FORMAT_W(5) " invoked implicitly", _implicit_concurrent); out->cr(); + out->print_cr(SIZE_FORMAT_W(5) " Completed Old GCs", _success_old_gcs); + out->print_cr(" " SIZE_FORMAT_W(5) " interruptions", _interrupted_old_gcs); + out->cr(); + out->print_cr(SIZE_FORMAT_W(5) " Degenerated GCs", _success_degenerated_gcs); out->print_cr(" " SIZE_FORMAT_W(5) " caused by allocation failure", _alloc_failure_degenerated); for (int c = 0; c < ShenandoahGC::_DEGENERATED_LIMIT; c++) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp index 5dfaf40ad920d..1d05e1a40a26c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp @@ -39,6 +39,8 @@ class ShenandoahTracer : public GCTracer { class ShenandoahCollectorPolicy : public CHeapObj { private: size_t _success_concurrent_gcs; + size_t _success_old_gcs; + size_t _interrupted_old_gcs; size_t _success_degenerated_gcs; size_t _success_full_gcs; size_t _alloc_failure_degenerated; @@ -64,6 +66,8 @@ class ShenandoahCollectorPolicy : public CHeapObj { void record_cycle_start(); void record_success_concurrent(); + void record_success_old(); + void record_interrupted_old(); void record_success_degenerated(); void record_success_full(); void record_alloc_failure_to_degenerated(ShenandoahGC::ShenandoahDegenPoint point); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index cc3704312233e..f8d902b10205d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -603,7 +603,8 @@ void ShenandoahConcurrentGC::op_init_mark() { // The current implementation of swap_remembered_set() copies the write-card-table // to the read-card-table. The remembered sets are also swapped for GLOBAL collections // so that the verifier works with the correct copy of the card table when verifying. - _generation->swap_remembered_set(); + ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_swap_rset); + _generation->swap_remembered_set(); } if (_generation->generation_mode() == GLOBAL) { @@ -612,6 +613,7 @@ void ShenandoahConcurrentGC::op_init_mark() { // Purge the SATB buffers, transferring any valid, old pointers to the // old generation mark queue. Any pointers in a young region will be // abandoned. + ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_transfer_satb); heap->transfer_old_pointers_from_satb(); } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index 910eff2f00ee4..8a849d8267f7f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -324,27 +324,7 @@ void ShenandoahControlThread::run_service() { global_heuristics->clear_metaspace_oom(); } - // Commit worker statistics to cycle data - heap->phase_timings()->flush_par_workers_to_cycle(); - if (ShenandoahPacing) { - heap->pacer()->flush_stats_to_cycle(); - } - - // Print GC stats for current cycle - { - LogTarget(Info, gc, stats) lt; - if (lt.is_enabled()) { - ResourceMark rm; - LogStream ls(lt); - heap->phase_timings()->print_cycle_on(&ls); - if (ShenandoahPacing) { - heap->pacer()->print_cycle_on(&ls); - } - } - } - - // Commit statistics to globals - heap->phase_timings()->flush_cycle_to_global(); + process_phase_timings(heap); // Print Metaspace change following GC (if logging is enabled). MetaspaceUtils::print_metaspace_change(meta_sizes); @@ -395,6 +375,31 @@ void ShenandoahControlThread::run_service() { } } +void ShenandoahControlThread::process_phase_timings(const ShenandoahHeap* heap) { + + // Commit worker statistics to cycle data + heap->phase_timings()->flush_par_workers_to_cycle(); + if (ShenandoahPacing) { + heap->pacer()->flush_stats_to_cycle(); + } + + // Print GC stats for current cycle + { + LogTarget(Info, gc, stats) lt; + if (lt.is_enabled()) { + ResourceMark rm; + LogStream ls(lt); + heap->phase_timings()->print_cycle_on(&ls); + if (ShenandoahPacing) { + heap->pacer()->print_cycle_on(&ls); + } + } + } + + // Commit statistics to globals + heap->phase_timings()->flush_cycle_to_global(); +} + // Young and old concurrent cycles are initiated by the regulator. Implicit // and explicit GC requests are handled by the controller thread and always // run a global cycle (which is concurrent by default, but may be overridden @@ -468,6 +473,9 @@ void ShenandoahControlThread::service_concurrent_old_cycle(const ShenandoahHeap* young_generation->set_mark_incomplete(); old_generation->set_mark_incomplete(); service_concurrent_cycle(young_generation, cause, true); + + process_phase_timings(heap); + if (heap->cancelled_gc()) { // Young generation bootstrap cycle has failed. Concurrent mark for old generation // is not going to resume after degenerated young cycle completes. @@ -478,11 +486,6 @@ void ShenandoahControlThread::service_concurrent_old_cycle(const ShenandoahHeap* // which has bootstrapped the old concurrent marking. _degen_point = ShenandoahGC::_degenerated_outside_cycle; - // TODO: Bit of a hack here to keep the phase timings happy as we transition - // to concurrent old marking. We need to revisit this. - heap->phase_timings()->flush_par_workers_to_cycle(); - heap->phase_timings()->flush_cycle_to_global(); - // From here we will 'resume' the old concurrent mark. This will skip reset // and init mark for the concurrent mark. All of that work will have been // done by the bootstrapping young cycle. In order to simplify the debugging @@ -531,7 +534,7 @@ void ShenandoahControlThread::resume_concurrent_old_cycle(ShenandoahGeneration* ShenandoahOldGC gc(generation, _allow_old_preemption); if (gc.collect(cause)) { generation->heuristics()->record_success_concurrent(); - heap->shenandoah_policy()->record_success_concurrent(); + heap->shenandoah_policy()->record_success_old(); } if (heap->cancelled_gc()) { @@ -545,6 +548,9 @@ void ShenandoahControlThread::resume_concurrent_old_cycle(ShenandoahGeneration* // cycle, then we are not actually going to a degenerated cycle, // so the degenerated point doesn't matter here. check_cancellation_or_degen(ShenandoahGC::_degenerated_outside_cycle); + if (_requested_gc_cause == GCCause::_shenandoah_concurrent_gc) { + heap->shenandoah_policy()->record_interrupted_old(); + } } } @@ -754,6 +760,7 @@ bool ShenandoahControlThread::request_concurrent_gc(GenerationMode generation) { _preemption_requested.set(); ShenandoahHeap::heap()->cancel_gc(GCCause::_shenandoah_concurrent_gc); notify_control_thread(); + MonitorLocker ml(&_regulator_lock, Mutex::_no_safepoint_check_flag); ml.wait(); return true; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp index e50b235b1077e..e0e22288d39a6 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp @@ -132,6 +132,8 @@ class ShenandoahControlThread: public ConcurrentGCThread { bool check_soft_max_changed() const; + void process_phase_timings(const ShenandoahHeap* heap); + public: // Constructor ShenandoahControlThread(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp index e913228fc4b38..e1f4dcc6aca61 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp @@ -53,6 +53,8 @@ class outputStream; f(init_mark_gross, "Pause Init Mark (G)") \ f(init_mark, "Pause Init Mark (N)") \ f(init_manage_tlabs, " Manage TLABs") \ + f(init_swap_rset, " Swap Remembered Set") \ + f(init_transfer_satb, " Transfer Old From SATB") \ f(init_update_region_states, " Update Region States") \ \ f(init_scan_rset, "Concurrent Scan Remembered Set") \ @@ -66,8 +68,6 @@ class outputStream; f(final_mark, "Pause Final Mark (N)") \ f(finish_mark, " Finish Mark") \ SHENANDOAH_PAR_PHASE_DO(finish_mark_, " FM: ", f) \ - f(coalesce_and_fill, "Coalesce and Fill Old Dead") \ - SHENANDOAH_PAR_PHASE_DO(coalesce_and_fill_, " CFOD: ", f) \ f(purge, " System Purge") \ SHENANDOAH_PAR_PHASE_DO(purge_cu_par_, " CU: ", f) \ f(purge_weak_par, " Weak Roots") \ @@ -100,6 +100,8 @@ class outputStream; f(conc_class_unload_purge_ec, " Exception Caches") \ f(conc_strong_roots, "Concurrent Strong Roots") \ SHENANDOAH_PAR_PHASE_DO(conc_strong_roots_, " CSR: ", f) \ + f(coalesce_and_fill, "Coalesce and Fill Old Dead") \ + SHENANDOAH_PAR_PHASE_DO(coalesce_and_fill_, " CFOD: ", f) \ f(conc_evac, "Concurrent Evacuation") \ \ f(final_roots_gross, "Pause Final Roots (G)") \ From 199e8080174a529dc8153f1196b767baa199d501 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Fri, 15 Apr 2022 23:19:48 +0000 Subject: [PATCH 123/254] Include mark worker performance in report for concurrent mark phase Reviewed-by: kdnilsen --- src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp | 3 ++- src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.cpp | 1 + src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp index beb46861d49b3..a7fa66da2b027 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp @@ -57,7 +57,8 @@ class ShenandoahConcurrentMarkingTask : public WorkerTask { void work(uint worker_id) { ShenandoahHeap* heap = ShenandoahHeap::heap(); - ShenandoahConcurrentWorkerSession worker_session(worker_id); + ShenandoahParallelWorkerSession worker_session(worker_id); + ShenandoahWorkerTimingsTracker timer(ShenandoahPhaseTimings::conc_mark, ShenandoahPhaseTimings::ParallelMark, worker_id); ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers); ShenandoahReferenceProcessor* rp = heap->active_generation()->ref_processor(); assert(rp != NULL, "need reference processor"); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.cpp b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.cpp index dc81575057d0f..a145abed7170b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.cpp @@ -113,6 +113,7 @@ bool ShenandoahPhaseTimings::is_worker_phase(Phase phase) { case degen_gc_purge_class_unload: case degen_gc_purge_weak_par: case heap_iteration_roots: + case conc_mark: case conc_mark_roots: case conc_thread_roots: case conc_weak_roots_work: diff --git a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp index e1f4dcc6aca61..9e1498a138f6f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp @@ -63,6 +63,7 @@ class outputStream; f(conc_mark_roots, "Concurrent Mark Roots ") \ SHENANDOAH_PAR_PHASE_DO(conc_mark_roots, " CMR: ", f) \ f(conc_mark, "Concurrent Marking") \ + SHENANDOAH_PAR_PHASE_DO(conc_mark, " CM: ", f) \ \ f(final_mark_gross, "Pause Final Mark (G)") \ f(final_mark, "Pause Final Mark (N)") \ From b759dab2c7d1f89cbb4e23ae331f828173acdf18 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Wed, 20 Apr 2022 15:52:05 +0000 Subject: [PATCH 124/254] Fix concurrent mark worker timings Reviewed-by: kdnilsen --- .../share/gc/shenandoah/shenandoahConcurrentMark.cpp | 2 +- src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.cpp | 6 +++--- src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp index a7fa66da2b027..5efc1916db965 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp @@ -58,7 +58,7 @@ class ShenandoahConcurrentMarkingTask : public WorkerTask { void work(uint worker_id) { ShenandoahHeap* heap = ShenandoahHeap::heap(); ShenandoahParallelWorkerSession worker_session(worker_id); - ShenandoahWorkerTimingsTracker timer(ShenandoahPhaseTimings::conc_mark, ShenandoahPhaseTimings::ParallelMark, worker_id); + ShenandoahWorkerTimingsTracker timer(ShenandoahPhaseTimings::conc_mark, ShenandoahPhaseTimings::ParallelMark, worker_id, true); ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers); ShenandoahReferenceProcessor* rp = heap->active_generation()->ref_processor(); assert(rp != NULL, "need reference processor"); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.cpp b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.cpp index a145abed7170b..8beaa8f1155b7 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.cpp @@ -310,17 +310,17 @@ void ShenandoahPhaseTimings::print_global_on(outputStream* out) const { } ShenandoahWorkerTimingsTracker::ShenandoahWorkerTimingsTracker(ShenandoahPhaseTimings::Phase phase, - ShenandoahPhaseTimings::ParPhase par_phase, uint worker_id) : + ShenandoahPhaseTimings::ParPhase par_phase, uint worker_id, bool cumulative) : _timings(ShenandoahHeap::heap()->phase_timings()), _phase(phase), _par_phase(par_phase), _worker_id(worker_id) { - assert(_timings->worker_data(_phase, _par_phase)->get(_worker_id) == ShenandoahWorkerData::uninitialized(), + assert(_timings->worker_data(_phase, _par_phase)->get(_worker_id) == ShenandoahWorkerData::uninitialized() || cumulative, "Should not be set yet: %s", ShenandoahPhaseTimings::phase_name(_timings->worker_par_phase(_phase, _par_phase))); _start_time = os::elapsedTime(); } ShenandoahWorkerTimingsTracker::~ShenandoahWorkerTimingsTracker() { - _timings->worker_data(_phase, _par_phase)->set(_worker_id, os::elapsedTime() - _start_time); + _timings->worker_data(_phase, _par_phase)->set_or_add(_worker_id, os::elapsedTime() - _start_time); if (ShenandoahPhaseTimings::is_root_work_phase(_phase)) { ShenandoahPhaseTimings::Phase root_phase = _phase; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp index 9e1498a138f6f..250eb40d93dd3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp @@ -259,7 +259,8 @@ class ShenandoahWorkerTimingsTracker : public StackObj { double _start_time; EventGCPhaseParallel _event; public: - ShenandoahWorkerTimingsTracker(ShenandoahPhaseTimings::Phase phase, ShenandoahPhaseTimings::ParPhase par_phase, uint worker_id); + ShenandoahWorkerTimingsTracker(ShenandoahPhaseTimings::Phase phase, ShenandoahPhaseTimings::ParPhase par_phase, + uint worker_id, bool cumulative = false); ~ShenandoahWorkerTimingsTracker(); }; From 5ec87a5da0a95135039ab2ec13659ffee3e8e551 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Wed, 20 Apr 2022 18:14:03 +0000 Subject: [PATCH 125/254] Use state from collection set to know if mixed collection is in progress Reviewed-by: kdnilsen --- .../shenandoah/heuristics/shenandoahHeuristics.cpp | 12 ++++-------- .../shenandoah/heuristics/shenandoahHeuristics.hpp | 3 +-- .../heuristics/shenandoahOldHeuristics.cpp | 3 +-- .../heuristics/shenandoahOldHeuristics.hpp | 3 +-- .../share/gc/shenandoah/shenandoahConcurrentGC.cpp | 4 +--- .../share/gc/shenandoah/shenandoahGeneration.cpp | 8 +++----- .../share/gc/shenandoah/shenandoahGeneration.hpp | 2 +- src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp | 14 +++++--------- src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp | 7 ------- .../gc/shenandoah/shenandoahOldGeneration.cpp | 3 +-- .../gc/shenandoah/shenandoahOldGeneration.hpp | 2 +- 11 files changed, 19 insertions(+), 42 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp index b534976a44c9c..d0ed5b4ddf4f7 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp @@ -77,9 +77,8 @@ ShenandoahHeuristics::~ShenandoahHeuristics() { FREE_C_HEAP_ARRAY(RegionGarbage, _region_data); } -// Returns true iff the chosen collection set includes old-gen regions -bool ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collection_set, ShenandoahOldHeuristics* old_heuristics) { - bool result = false; +void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collection_set, ShenandoahOldHeuristics* old_heuristics) { + ShenandoahHeap* heap = ShenandoahHeap::heap(); assert(collection_set->count() == 0, "Must be empty"); @@ -184,9 +183,7 @@ bool ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec if (immediate_percent <= ShenandoahImmediateThreshold) { if (old_heuristics != NULL) { - if (old_heuristics->prime_collection_set(collection_set)) { - result = true; - } + old_heuristics->prime_collection_set(collection_set); size_t bytes_reserved_for_old_evacuation = collection_set->get_old_bytes_reserved_for_evacuation(); if (bytes_reserved_for_old_evacuation * ShenandoahEvacWaste < heap->get_old_evac_reserve()) { @@ -206,7 +203,7 @@ bool ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec // we'll need to borrow from old-gen in order to evacuate. If there's nothing to borrow, we're going to // degenerate to full GC. - // TODO: younng_available can include available (between top() and end()) within each young region that is not + // TODO: young_available can include available (between top() and end()) within each young region that is not // part of the collection set. Making this memory available to the young_evacuation_reserve allows a larger // young collection set to be chosen when available memory is under extreme pressure. Implementing this "improvement" // is tricky, because the incremental construction of the collection set actually changes the amount of memory @@ -356,7 +353,6 @@ bool ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec byte_size_in_proper_unit(collection_set->garbage()), proper_unit_for_byte_size(collection_set->garbage()), cset_percent); - return result; } void ShenandoahHeuristics::record_cycle_start() { diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp index aa96c81d01a7a..a2a3a5f370255 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp @@ -153,8 +153,7 @@ class ShenandoahHeuristics : public CHeapObj { virtual void record_requested_gc(); - // Return true iff the chosen collection set includes at least one old-gen region. - virtual bool choose_collection_set(ShenandoahCollectionSet* collection_set, ShenandoahOldHeuristics* old_heuristics); + virtual void choose_collection_set(ShenandoahCollectionSet* collection_set, ShenandoahOldHeuristics* old_heuristics); virtual bool can_unload_classes(); virtual bool can_unload_classes_normal(); diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp index 395c255bb78ad..d7791a990a234 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp @@ -153,14 +153,13 @@ bool ShenandoahOldHeuristics::prime_collection_set(ShenandoahCollectionSet* coll } // Both arguments are don't cares for old-gen collections -bool ShenandoahOldHeuristics::choose_collection_set(ShenandoahCollectionSet* collection_set, +void ShenandoahOldHeuristics::choose_collection_set(ShenandoahCollectionSet* collection_set, ShenandoahOldHeuristics* old_heuristics) { assert((collection_set == nullptr) && (old_heuristics == nullptr), "Expect null arguments in ShenandoahOldHeuristics::choose_collection_set()"); // Old-gen doesn't actually choose a collection set to be evacuated by its own gang of worker tasks. // Instead, it computes the set of regions to be evacuated by subsequent young-gen evacuation passes. prepare_for_old_collections(); - return false; } void ShenandoahOldHeuristics::prepare_for_old_collections() { diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp index 57823881196fc..46d4fea79ccc9 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp @@ -79,8 +79,7 @@ class ShenandoahOldHeuristics : public ShenandoahHeuristics { public: ShenandoahOldHeuristics(ShenandoahGeneration* generation, ShenandoahHeuristics* trigger_heuristic); - // Return true iff chosen collection set includes at least one old-gen HeapRegion. - virtual bool choose_collection_set(ShenandoahCollectionSet* collection_set, ShenandoahOldHeuristics* old_heuristics) override; + virtual void choose_collection_set(ShenandoahCollectionSet* collection_set, ShenandoahOldHeuristics* old_heuristics) override; // Return true iff the collection set is primed with at least one old-gen region. bool prime_collection_set(ShenandoahCollectionSet* set); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index f8d902b10205d..4a1c42d93a289 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -694,7 +694,7 @@ void ShenandoahConcurrentGC::op_final_mark() { // are negligible negative impacts from delaying completion of old-gen evacuation) and when young-gen collections // are "under duress" (as signalled by very low availability of memory within young-gen, indicating that/ young-gen // collections are not triggering frequently enough). - bool mixed_evac = _generation->prepare_regions_and_collection_set(true /*concurrent*/); + _generation->prepare_regions_and_collection_set(true /*concurrent*/); // Upon return from prepare_regions_and_collection_set(), certain parameters have been established to govern the // evacuation efforts that are about to begin. In particular: @@ -716,8 +716,6 @@ void ShenandoahConcurrentGC::op_final_mark() { // heap->get_alloc_supplement_reserve() represents the amount of old-gen memory that can be allocated during evacuation // and update-refs phases of gc. The young evacuation reserve has already been removed from this quantity. - heap->set_mixed_evac(mixed_evac); - // Has to be done after cset selection heap->prepare_concurrent_roots(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index 51f95d828992b..50e4b4f8c92ce 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -221,9 +221,8 @@ void ShenandoahGeneration::prepare_gc(bool do_old_gc_bootstrap) { } } -// Returns true iff the chosen collection set includes a mix of young-gen and old-gen regions. -bool ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) { - bool result; +void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) { + ShenandoahHeap* heap = ShenandoahHeap::heap(); assert(!heap->is_full_gc_in_progress(), "Only for concurrent and degenerated GC"); assert(generation_mode() != OLD, "Only YOUNG and GLOBAL GC perform evacuations"); @@ -333,7 +332,7 @@ bool ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) { // The heuristics may consult and/or change the values of PromotionReserved, OldEvacuationReserved, and // YoungEvacuationReserved, all of which are represented in the shared ShenandoahHeap data structure. - result = _heuristics->choose_collection_set(heap->collection_set(), heap->old_heuristics()); + _heuristics->choose_collection_set(heap->collection_set(), heap->old_heuristics()); // EvacuationAllocationSupplement: This represents memory that can be allocated in excess of young_gen->available() // during evacuation and update-refs. This memory can be temporarily borrowed from old-gen allotment, then @@ -359,7 +358,6 @@ bool ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) { ShenandoahHeapLocker locker(heap->lock()); heap->free_set()->rebuild(); } - return result; } bool ShenandoahGeneration::is_bitmap_clear() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index 5669ffc12bdf4..ebeda4f9cb4d8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -112,7 +112,7 @@ class ShenandoahGeneration : public CHeapObj { void prepare_gc(bool do_old_gc_bootstrap); // Return true iff prepared collection set includes at least one old-gen HeapRegion. - virtual bool prepare_regions_and_collection_set(bool concurrent); + virtual void prepare_regions_and_collection_set(bool concurrent); // Cancel marking (used by Full collect and when cancelling cycle). virtual void cancel_marking(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 61bb1e2d3e441..1910001a7b18c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -497,9 +497,7 @@ void ShenandoahHeap::initialize_heuristics() { ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) : CollectedHeap(), _gc_generation(NULL), - _mixed_evac(false), _prep_for_mixed_evac_in_progress(false), - _evacuation_allowance(0), _initial_size(0), _used(0), _committed(0), @@ -2405,14 +2403,12 @@ class ShenandoahUpdateHeapRefsTask : public WorkerTask { private: ShenandoahHeap* _heap; ShenandoahRegionIterator* _regions; - bool _mixed_evac; // true iff most recent evacuation includes old-gen HeapRegions public: - ShenandoahUpdateHeapRefsTask(ShenandoahRegionIterator* regions, bool mixed_evac) : + explicit ShenandoahUpdateHeapRefsTask(ShenandoahRegionIterator* regions) : WorkerTask("Shenandoah Update References"), _heap(ShenandoahHeap::heap()), - _regions(regions), - _mixed_evac(mixed_evac) + _regions(regions) { } @@ -2450,7 +2446,7 @@ class ShenandoahUpdateHeapRefsTask : public WorkerTask { _heap->marked_object_oop_iterate(r, &cl, update_watermark); } else { // Old region in a young cycle or mixed cycle. - if (!_mixed_evac) { + if (!is_mixed) { // This is a young evac.. _heap->card_scan()->process_region(r, &cl, true); } else { @@ -2526,10 +2522,10 @@ void ShenandoahHeap::update_heap_references(bool concurrent) { assert(!is_full_gc_in_progress(), "Only for concurrent and degenerated GC"); if (concurrent) { - ShenandoahUpdateHeapRefsTask task(&_update_refs_iterator, _mixed_evac); + ShenandoahUpdateHeapRefsTask task(&_update_refs_iterator); workers()->run_task(&task); } else { - ShenandoahUpdateHeapRefsTask task(&_update_refs_iterator, _mixed_evac); + ShenandoahUpdateHeapRefsTask task(&_update_refs_iterator); workers()->run_task(&task); } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index 8ac2bfffd3896..318582d4a1b64 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -151,11 +151,8 @@ class ShenandoahHeap : public CollectedHeap { ShenandoahHeapLock _lock; ShenandoahGeneration* _gc_generation; - bool _mixed_evac; // true iff most recent evac included at least one old-gen HeapRegion bool _prep_for_mixed_evac_in_progress; // true iff we are concurrently coalescing and filling old-gen HeapRegions - size_t _evacuation_allowance; // amount by which young-gen usage may temporarily exceed young-gen capacity - public: ShenandoahHeapLock* lock() { return &_lock; @@ -170,10 +167,6 @@ class ShenandoahHeap : public CollectedHeap { _gc_generation = generation; } - void set_mixed_evac(bool mixed_evac) { - _mixed_evac = mixed_evac; - } - ShenandoahOldHeuristics* old_heuristics(); bool doing_mixed_evacuations(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp index 6e1ff76568d83..e4dbd8889040f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp @@ -181,7 +181,7 @@ bool ShenandoahOldGeneration::contains(oop obj) const { return ShenandoahHeap::heap()->is_in_old(obj); } -bool ShenandoahOldGeneration::prepare_regions_and_collection_set(bool concurrent) { +void ShenandoahOldGeneration::prepare_regions_and_collection_set(bool concurrent) { ShenandoahHeap* heap = ShenandoahHeap::heap(); assert(!heap->is_full_gc_in_progress(), "Only for concurrent and degenerated GC"); @@ -204,7 +204,6 @@ bool ShenandoahOldGeneration::prepare_regions_and_collection_set(bool concurrent ShenandoahHeapLocker locker(heap->lock()); heap->free_set()->rebuild(); } - return false; } ShenandoahHeuristics* ShenandoahOldGeneration::initialize_heuristics(ShenandoahMode* gc_mode) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp index 49dae3e75c156..39e2656e2deb3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp @@ -48,7 +48,7 @@ class ShenandoahOldGeneration : public ShenandoahGeneration { virtual void cancel_marking() override; - bool prepare_regions_and_collection_set(bool concurrent) override; + void prepare_regions_and_collection_set(bool concurrent) override; virtual ShenandoahHeuristics* initialize_heuristics(ShenandoahMode* gc_mode) override; From bdd7667ad0dcfd46871edcfbe67aade6b27d1455 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Fri, 22 Apr 2022 17:07:26 +0000 Subject: [PATCH 126/254] Only clear requested gc cause when it is handled Reviewed-by: kdnilsen --- .../mode/shenandoahGenerationalMode.cpp | 14 ++++++++++++++ .../mode/shenandoahGenerationalMode.hpp | 1 + .../gc/shenandoah/shenandoahControlThread.cpp | 18 +++++++++++------- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp index 773b38ebe476d..eca823526bf78 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp @@ -85,3 +85,17 @@ const char* affiliation_name(ShenandoahRegionAffiliation type) { return nullptr; } } + +const char* generation_name(GenerationMode mode) { + switch (mode) { + case GenerationMode::GLOBAL: + return "Global"; + case GenerationMode::OLD: + return "Old"; + case GenerationMode::YOUNG: + return "Young"; + default: + ShouldNotReachHere(); + return nullptr; + } +} diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp index 5a5fefd98e804..6527a3b2c41c4 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp @@ -43,6 +43,7 @@ enum ShenandoahRegionAffiliation { const char* affiliation_name(oop ptr); const char* affiliation_name(ShenandoahRegionAffiliation type); const char affiliation_code(ShenandoahRegionAffiliation type); +const char* generation_name(GenerationMode mode); class ShenandoahGenerationalMode : public ShenandoahMode { public: diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index 8a849d8267f7f..e7c418097ba0c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -210,19 +210,23 @@ void ShenandoahControlThread::run_service() { } else { heap->set_unload_classes(false); } + + // Don't want to spin in this loop and start a cycle every time, so + // clear requested gc cause. This creates a race with callers of the + // blocking 'request_gc' method, but there it loops and resets the + // '_requested_gc_cause' until a full cycle is completed. + _requested_gc_cause = GCCause::_no_gc; } else if (heap->is_concurrent_old_mark_in_progress() || heap->is_concurrent_prep_for_mixed_evacuation_in_progress()) { // Nobody asked us to do anything, but we have an old-generation mark or old-generation preparation for // mixed evacuation in progress, so resume working on that. + log_info(gc)("Resume old gc: marking=%s, preparing=%s", + BOOL_TO_STR(heap->is_concurrent_old_mark_in_progress()), + BOOL_TO_STR(heap->is_concurrent_prep_for_mixed_evacuation_in_progress())); + cause = GCCause::_shenandoah_concurrent_gc; generation = OLD; set_gc_mode(marking_old); } - - // Don't want to spin in this loop and start a cycle every time, so - // clear requested gc cause. This creates a race with callers of the - // blocking 'request_gc' method, but there it loops and resets the - // '_requested_gc_cause' until a full cycle is completed. - _requested_gc_cause = GCCause::_no_gc; } // Blow all soft references on this cycle, if handling allocation failure, @@ -754,7 +758,7 @@ bool ShenandoahControlThread::request_concurrent_gc(GenerationMode generation) { } if (preempt_old_marking(generation)) { - log_info(gc)("Preempting old generation mark to allow young GC."); + log_info(gc)("Preempting old generation mark to allow %s GC.", generation_name(generation)); _requested_gc_cause = GCCause::_shenandoah_concurrent_gc; _requested_generation = generation; _preemption_requested.set(); From b4ad3540ca9f5ca04ab034bc17a4e0c9f0dec2ad Mon Sep 17 00:00:00 2001 From: William Kemper Date: Fri, 29 Apr 2022 15:27:43 +0000 Subject: [PATCH 127/254] Allow regulator to schedule global cycles for the purpose of class unloading Reviewed-by: kdnilsen --- .../mode/shenandoahGenerationalMode.cpp | 2 +- .../shenandoah/shenandoahRegulatorThread.cpp | 22 ++++++++++++++----- .../shenandoah/shenandoahRegulatorThread.hpp | 2 ++ 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp index eca823526bf78..f33bf97d2fe5b 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp @@ -32,11 +32,11 @@ void ShenandoahGenerationalMode::initialize_flags() const { if (ClassUnloading) { - // Leaving this here for the day we re-enable class unloading FLAG_SET_DEFAULT(ShenandoahSuspendibleWorkers, true); FLAG_SET_DEFAULT(VerifyBeforeExit, false); } + SHENANDOAH_ERGO_OVERRIDE_DEFAULT(ShenandoahUnloadClassesFrequency, 0); SHENANDOAH_ERGO_ENABLE_FLAG(ExplicitGCInvokesConcurrent); SHENANDOAH_ERGO_ENABLE_FLAG(ShenandoahImplicitGCInvokesConcurrent); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.cpp index a80a890e4f1e1..c193b46aa3845 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.cpp @@ -71,14 +71,20 @@ void ShenandoahRegulatorThread::regulate_concurrent_cycles() { while (!should_terminate()) { ShenandoahControlThread::GCMode mode = _control_thread->gc_mode(); if (mode == ShenandoahControlThread::none) { - if (start_old_cycle()) { - log_info(gc)("Heuristics request for old collection accepted"); - } else if (start_young_cycle()) { - log_info(gc)("Heuristics request for young collection accepted"); + if (should_unload_classes()) { + if (_control_thread->request_concurrent_gc(GLOBAL)) { + log_info(gc)("Heuristics request for global (unload classes) accepted."); + } + } else { + if (start_old_cycle()) { + log_info(gc)("Heuristics request for old collection accepted"); + } else if (start_young_cycle()) { + log_info(gc)("Heuristics request for young collection accepted"); + } } } else if (mode == ShenandoahControlThread::marking_old) { if (start_young_cycle()) { - log_info(gc)("Heuristics request for young collection accepted"); + log_info(gc)("Heuristics request to interrupt old for young collection accepted"); } } @@ -149,3 +155,9 @@ void ShenandoahRegulatorThread::stop_service() { log_info(gc)("%s: Stop requested.", name()); } +bool ShenandoahRegulatorThread::should_unload_classes() { + // The heuristics delegate this decision to the collector policy, which is based on the number + // of cycles started. + return _global_heuristics->should_unload_classes(); +} + diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.hpp b/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.hpp index b29ba8f0a06a3..a07ae7ded3fff 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.hpp @@ -73,6 +73,8 @@ class ShenandoahRegulatorThread: public ConcurrentGCThread { bool start_young_cycle(); bool start_global_cycle(); + bool should_unload_classes(); + ShenandoahSharedFlag _heap_changed; ShenandoahControlThread* _control_thread; ShenandoahHeuristics* _young_heuristics; From 10ebe4e429dcf578376b3bd7f61d4f02449c0a35 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Fri, 29 Apr 2022 15:28:32 +0000 Subject: [PATCH 128/254] Include mixed and abbreviated cycle counts in summary report Reviewed-by: kdnilsen --- .../heuristics/shenandoahHeuristics.cpp | 8 +++++++- .../gc/shenandoah/shenandoahCollectorPolicy.cpp | 17 ++++++++++++++++- .../gc/shenandoah/shenandoahCollectorPolicy.hpp | 4 ++++ 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp index d0ed5b4ddf4f7..47958d329a0bd 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp @@ -331,8 +331,14 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec byte_size_in_proper_unit(alloc_budget_evac_and_update), proper_unit_for_byte_size(alloc_budget_evac_and_update), byte_size_in_proper_unit(potential_evac_supplement), proper_unit_for_byte_size(potential_evac_supplement)); + } else { + // we're going to skip evacuation and update refs because we reclaimed sufficient amounts of immediate garbage. + heap->shenandoah_policy()->record_abbreviated_cycle(); + } + + if (collection_set->has_old_regions()) { + heap->shenandoah_policy()->record_mixed_cycle(); } - // else, we're going to skip evacuation and update refs because we reclaimed sufficient amounts of immediate garbage. size_t cset_percent = (total_garbage == 0) ? 0 : (collection_set->garbage() * 100 / total_garbage); size_t collectable_garbage = collection_set->garbage() + immediate_garbage; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.cpp index 1cfd2a88a3f5a..ae6b211315693 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.cpp @@ -31,6 +31,8 @@ ShenandoahCollectorPolicy::ShenandoahCollectorPolicy() : _success_concurrent_gcs(0), + _mixed_gcs(0), + _abbreviated_cycles(0), _success_old_gcs(0), _interrupted_old_gcs(0), _success_degenerated_gcs(0), @@ -85,6 +87,14 @@ void ShenandoahCollectorPolicy::record_success_concurrent() { _success_concurrent_gcs++; } +void ShenandoahCollectorPolicy::record_mixed_cycle() { + _mixed_gcs++; +} + +void ShenandoahCollectorPolicy::record_abbreviated_cycle() { + _abbreviated_cycles++; +} + void ShenandoahCollectorPolicy::record_success_old() { _success_old_gcs++; } @@ -121,7 +131,8 @@ void ShenandoahCollectorPolicy::print_gc_stats(outputStream* out) const { out->print_cr("Under allocation pressure, concurrent cycles may cancel, and either continue cycle"); out->print_cr("under stop-the-world pause or result in stop-the-world Full GC. Increase heap size,"); out->print_cr("tune GC heuristics, set more aggressive pacing delay, or lower allocation rate"); - out->print_cr("to avoid Degenerated and Full GC cycles."); + out->print_cr("to avoid Degenerated and Full GC cycles. Abbreviated cycles are those which found"); + out->print_cr("enough regions with no live objects to skip evacuation."); out->cr(); out->print_cr(SIZE_FORMAT_W(5) " Successful Concurrent GCs", _success_concurrent_gcs); @@ -130,6 +141,7 @@ void ShenandoahCollectorPolicy::print_gc_stats(outputStream* out) const { out->cr(); out->print_cr(SIZE_FORMAT_W(5) " Completed Old GCs", _success_old_gcs); + out->print_cr(" " SIZE_FORMAT_W(5) " mixed", _mixed_gcs); out->print_cr(" " SIZE_FORMAT_W(5) " interruptions", _interrupted_old_gcs); out->cr(); @@ -144,6 +156,9 @@ void ShenandoahCollectorPolicy::print_gc_stats(outputStream* out) const { out->print_cr(" " SIZE_FORMAT_W(5) " upgraded to Full GC", _alloc_failure_degenerated_upgrade_to_full); out->cr(); + out->print_cr(SIZE_FORMAT_W(5) " Abbreviated GCs", _abbreviated_cycles); + out->cr(); + out->print_cr(SIZE_FORMAT_W(5) " Full GCs", _success_full_gcs + _alloc_failure_degenerated_upgrade_to_full); out->print_cr(" " SIZE_FORMAT_W(5) " invoked explicitly", _explicit_full); out->print_cr(" " SIZE_FORMAT_W(5) " invoked implicitly", _implicit_full); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp index 1d05e1a40a26c..b9ada595819a5 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp @@ -39,6 +39,8 @@ class ShenandoahTracer : public GCTracer { class ShenandoahCollectorPolicy : public CHeapObj { private: size_t _success_concurrent_gcs; + size_t _mixed_gcs; + size_t _abbreviated_cycles; size_t _success_old_gcs; size_t _interrupted_old_gcs; size_t _success_degenerated_gcs; @@ -65,6 +67,8 @@ class ShenandoahCollectorPolicy : public CHeapObj { // These two encompass the entire cycle. void record_cycle_start(); + void record_mixed_cycle(); + void record_abbreviated_cycle(); void record_success_concurrent(); void record_success_old(); void record_interrupted_old(); From 469df3ec494e5533cd0141780b47e9a5383c6876 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Wed, 4 May 2022 20:41:18 +0000 Subject: [PATCH 129/254] Have adaptive heuristic include only full cycles in average cycle time Reviewed-by: zgu --- .../heuristics/shenandoahAdaptiveHeuristics.cpp | 4 ++-- .../heuristics/shenandoahAdaptiveHeuristics.hpp | 2 +- .../gc/shenandoah/heuristics/shenandoahHeuristics.cpp | 8 +++++--- .../gc/shenandoah/heuristics/shenandoahHeuristics.hpp | 2 +- .../shenandoah/heuristics/shenandoahOldHeuristics.cpp | 4 ++-- .../shenandoah/heuristics/shenandoahOldHeuristics.hpp | 2 +- .../share/gc/shenandoah/shenandoahConcurrentGC.cpp | 4 +++- .../share/gc/shenandoah/shenandoahConcurrentGC.hpp | 3 ++- .../share/gc/shenandoah/shenandoahControlThread.cpp | 4 ++-- src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp | 10 ++++++++++ 10 files changed, 29 insertions(+), 14 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp index d5302ef872370..53667a39f44d0 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp @@ -150,8 +150,8 @@ void ShenandoahAdaptiveHeuristics::record_cycle_start() { _allocation_rate.allocation_counter_reset(); } -void ShenandoahAdaptiveHeuristics::record_success_concurrent() { - ShenandoahHeuristics::record_success_concurrent(); +void ShenandoahAdaptiveHeuristics::record_success_concurrent(bool abbreviated) { + ShenandoahHeuristics::record_success_concurrent(abbreviated); size_t available = ShenandoahHeap::heap()->free_set()->available(); diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp index babedaaa24274..45655ef388ad7 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp @@ -62,7 +62,7 @@ class ShenandoahAdaptiveHeuristics : public ShenandoahHeuristics { size_t actual_free); void record_cycle_start(); - void record_success_concurrent(); + void record_success_concurrent(bool abbreviated); void record_success_degenerated(); void record_success_full(); diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp index 47958d329a0bd..a87d1c04785af 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp @@ -410,12 +410,14 @@ void ShenandoahHeuristics::adjust_penalty(intx step) { "In range after adjustment: " INTX_FORMAT, _gc_time_penalties); } -void ShenandoahHeuristics::record_success_concurrent() { +void ShenandoahHeuristics::record_success_concurrent(bool abbreviated) { _degenerated_cycles_in_a_row = 0; _successful_cycles_in_a_row++; - _gc_time_history->add(time_since_last_gc()); - _gc_times_learned++; + if (!(abbreviated && ShenandoahAdaptiveIgnoreShortCycles)) { + _gc_time_history->add(time_since_last_gc()); + _gc_times_learned++; + } adjust_penalty(Concurrent_Adjust); } diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp index a2a3a5f370255..a3f088863665d 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp @@ -143,7 +143,7 @@ class ShenandoahHeuristics : public CHeapObj { virtual bool should_degenerate_cycle(); - virtual void record_success_concurrent(); + virtual void record_success_concurrent(bool abbreviated); virtual void record_success_degenerated(); diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp index d7791a990a234..160830659af14 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp @@ -348,8 +348,8 @@ bool ShenandoahOldHeuristics::should_degenerate_cycle() { return _trigger_heuristic->should_degenerate_cycle(); } -void ShenandoahOldHeuristics::record_success_concurrent() { - _trigger_heuristic->record_success_concurrent(); +void ShenandoahOldHeuristics::record_success_concurrent(bool abbreviated) { + _trigger_heuristic->record_success_concurrent(abbreviated); } void ShenandoahOldHeuristics::record_success_degenerated() { diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp index 46d4fea79ccc9..2f77aa137692c 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp @@ -129,7 +129,7 @@ class ShenandoahOldHeuristics : public ShenandoahHeuristics { virtual bool should_degenerate_cycle() override; - virtual void record_success_concurrent() override; + virtual void record_success_concurrent(bool abbreviated) override; virtual void record_success_degenerated() override; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index 4a1c42d93a289..1d03dc9846fe6 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -90,7 +90,7 @@ class ShenandoahBreakpointMarkScope : public StackObj { ShenandoahConcurrentGC::ShenandoahConcurrentGC(ShenandoahGeneration* generation, bool do_old_gc_bootstrap) : _mark(generation), _degen_point(ShenandoahDegenPoint::_degenerated_unset), - _mixed_evac (false), + _abbreviated(false), _do_old_gc_bootstrap(do_old_gc_bootstrap), _generation(generation) { } @@ -214,7 +214,9 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) { } else { // We chose not to evacuate because we found sufficient immediate garbage. vmop_entry_final_roots(heap->is_aging_cycle()); + _abbreviated = true; } + size_t old_available, young_available; { ShenandoahYoungGeneration* young_gen = heap->young_generation(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp index c82408b7767dc..f0de2a9696528 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp @@ -49,7 +49,7 @@ class ShenandoahConcurrentGC : public ShenandoahGC { private: ShenandoahDegenPoint _degen_point; - bool _mixed_evac; // true iff most recent evacuation includes old-gen HeapRegions + bool _abbreviated; const bool _do_old_gc_bootstrap; protected: @@ -59,6 +59,7 @@ class ShenandoahConcurrentGC : public ShenandoahGC { ShenandoahConcurrentGC(ShenandoahGeneration* generation, bool do_old_gc_bootstrap); bool collect(GCCause::Cause cause); ShenandoahDegenPoint degen_point() const; + bool abbreviated() const { return _abbreviated; } private: // Entry points to STW GC operations, these cause a related safepoint, that then diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index e7c418097ba0c..58f803473af65 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -537,7 +537,7 @@ void ShenandoahControlThread::resume_concurrent_old_cycle(ShenandoahGeneration* // is allowed to cancel a GC. ShenandoahOldGC gc(generation, _allow_old_preemption); if (gc.collect(cause)) { - generation->heuristics()->record_success_concurrent(); + generation->heuristics()->record_success_concurrent(false); heap->shenandoah_policy()->record_success_old(); } @@ -605,7 +605,7 @@ void ShenandoahControlThread::service_concurrent_cycle(ShenandoahGeneration* gen ShenandoahConcurrentGC gc(generation, do_old_gc_bootstrap); if (gc.collect(cause)) { // Cycle is complete - generation->heuristics()->record_success_concurrent(); + generation->heuristics()->record_success_concurrent(gc.abbreviated()); heap->shenandoah_policy()->record_success_concurrent(); } else { assert(heap->cancelled_gc(), "Must have been cancelled"); diff --git a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp index e783a2ff32518..c532f89608243 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp @@ -170,6 +170,16 @@ "Larger values give more weight to recent values.") \ range(0,1.0) \ \ + product(bool, ShenandoahAdaptiveIgnoreShortCycles, true, EXPERIMENTAL, \ + "The adaptive heuristic tracks a moving average of cycle " \ + "times in order to start a gc before memory is exhausted. " \ + "In some cases, Shenandoah may skip the evacuation and update " \ + "reference phases, resulting in a shorter cycle. These may skew " \ + "the average cycle time downward and may cause the heuristic " \ + "to wait too long to start a cycle. Disabling this will have " \ + "the gc run less often, which will reduce CPU utilization, but" \ + "increase the risk of degenerated cycles.") \ + \ product(uintx, ShenandoahGuaranteedGCInterval, 5*60*1000, EXPERIMENTAL, \ "Many heuristics would guarantee a concurrent GC cycle at " \ "least with this interval. This is useful when large idle " \ From 58ea1065304aaf45d51069a2e0ee7a3e71007677 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Thu, 12 May 2022 16:13:18 +0000 Subject: [PATCH 130/254] Filter out invalid pointers picked up by old gen SATB Reviewed-by: kdnilsen --- .../gc/shenandoah/shenandoahConcurrentGC.cpp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index 1d03dc9846fe6..53d152623d7d9 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -1140,6 +1140,24 @@ void ShenandoahConcurrentGC::op_final_updaterefs() { heap->verifier()->verify_roots_in_to_space(); } + if (heap->mode()->is_generational() && heap->is_concurrent_old_mark_in_progress()) { + // When the SATB barrier is left on to support concurrent old gen mark, it may pick up writes to + // objects in the collection set. After those objects are evacuated, the pointers in the + // SATB are no longer safe. Once we have finished update references, we are guaranteed that + // no more writes to the collection set are possible. + // + // This will transfer any old pointers in _active_ regions from the SATB to the old gen + // mark queues. All other pointers will be discarded. This would also discard any pointers + // in old regions that were included in a mixed evacuation. We aren't using the SATB filter + // methods here because we cannot control when they execute. If the SATB filter runs _after_ + // a region has been recycled, we will not be able to detect the bad pointer. + // + // We are not concerned about skipping this step in abbreviated cycles because regions + // with no live objects cannot have been written to and so cannot have entries in the SATB + // buffers. + heap->transfer_old_pointers_from_satb(); + } + heap->update_heap_region_states(true /*concurrent*/); heap->set_update_refs_in_progress(false); From 5473b0791974077e47ebad400e537b7f5e920912 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Wed, 18 May 2022 17:52:43 +0000 Subject: [PATCH 131/254] Start young collect instead of old if mixed evacuations are pending Reviewed-by: kdnilsen --- .../share/gc/shenandoah/shenandoahControlThread.cpp | 11 ++++++++++- src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp | 1 + 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index 58f803473af65..12b9487b03b4a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -195,9 +195,18 @@ void ShenandoahControlThread::run_service() { // We should only be here if the regulator requested a cycle or if // there is an old generation mark in progress. if (_requested_gc_cause == GCCause::_shenandoah_concurrent_gc) { + if (_requested_generation == OLD && heap->doing_mixed_evacuations()) { + // If a request to start an old cycle arrived while an old cycle was running, but _before_ + // it chose any regions for evacuation we don't want to start a new old cycle. Rather, we want + // the heuristic to run a young collection so that we can evacuate some old regions. + assert(!heap->is_concurrent_old_mark_in_progress(), "Should not be running mixed collections and concurrent marking."); + generation = YOUNG; + } else { + generation = _requested_generation; + } + // preemption was requested or this is a regular cycle cause = GCCause::_shenandoah_concurrent_gc; - generation = _requested_generation; set_gc_mode(default_mode); // Don't start a new old marking if there is one already in progress. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp index de70a99c1b8c2..5010163715cb0 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp @@ -132,6 +132,7 @@ void ShenandoahOldGC::op_final_mark() { bool ShenandoahOldGC::collect(GCCause::Cause cause) { ShenandoahHeap* heap = ShenandoahHeap::heap(); + assert(!heap->doing_mixed_evacuations(), "Should not start an old gc with pending mixed evacuations"); if (!heap->is_concurrent_prep_for_mixed_evacuation_in_progress()) { // Skip over the initial phases of old collect if we're resuming mixed evacuation preparation. From ea3191de45e97a4ad4f91f507dc85de0e51a04af Mon Sep 17 00:00:00 2001 From: William Kemper Date: Thu, 9 Jun 2022 18:22:49 +0000 Subject: [PATCH 132/254] Reserve regions for the collector according to generational reserves Reviewed-by: kdnilsen --- .../share/gc/shenandoah/shenandoahFreeSet.cpp | 19 +++++++++++++++---- .../share/gc/shenandoah/shenandoahFreeSet.hpp | 2 ++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index 312d87a38a8a1..37bdfd30032ae 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -573,7 +573,21 @@ void ShenandoahFreeSet::rebuild() { } // Evac reserve: reserve trailing space for evacuations - size_t to_reserve = (_heap->max_capacity() / 100) * ShenandoahEvacReserve; + if (!_heap->mode()->is_generational()) { + size_t to_reserve = (_heap->max_capacity() / 100) * ShenandoahEvacReserve; + reserve_regions(to_reserve); + } else { + size_t young_reserve = (_heap->young_generation()->max_capacity() / 100) * ShenandoahEvacReserve; + size_t old_reserve = (_heap->old_generation()->max_capacity() / 100) * ShenandoahOldEvacReserve; + size_t to_reserve = young_reserve + old_reserve; + reserve_regions(to_reserve); + } + + recompute_bounds(); + assert_bounds(); +} + +void ShenandoahFreeSet::reserve_regions(size_t to_reserve) { size_t reserved = 0; for (size_t idx = _heap->num_regions() - 1; idx > 0; idx--) { @@ -589,9 +603,6 @@ void ShenandoahFreeSet::rebuild() { log_debug(gc)(" Shifting region " SIZE_FORMAT " from mutator_free to collector_free", idx); } } - - recompute_bounds(); - assert_bounds(); } void ShenandoahFreeSet::log_status() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp index 98a9a66f22e44..153c759cc7a93 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp @@ -105,6 +105,8 @@ class ShenandoahFreeSet : public CHeapObj { double external_fragmentation(); void print_on(outputStream* out) const; + + void reserve_regions(size_t to_reserve); }; #endif // SHARE_GC_SHENANDOAH_SHENANDOAHFREESET_HPP From a8676f6d13e39fc8452047cdde3621c792f824c0 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Mon, 13 Jun 2022 17:20:23 +0000 Subject: [PATCH 133/254] Fix fullgc Reviewed-by: wkemper --- .../gc/shenandoah/shenandoahConcurrentGC.cpp | 55 ++++++++++--------- .../gc/shenandoah/shenandoahDegeneratedGC.cpp | 8 +++ .../share/gc/shenandoah/shenandoahFullGC.cpp | 40 +++++++++----- .../gc/shenandoah/shenandoahHeapRegion.cpp | 26 +++++++-- .../gc/shenandoah/shenandoahHeapRegion.hpp | 1 + 5 files changed, 85 insertions(+), 45 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index 53d152623d7d9..998cf82c0f6c4 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -217,33 +217,36 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) { _abbreviated = true; } - size_t old_available, young_available; - { - ShenandoahYoungGeneration* young_gen = heap->young_generation(); - ShenandoahGeneration* old_gen = heap->old_generation(); - ShenandoahHeapLocker locker(heap->lock()); - - size_t old_usage_before_evac = heap->capture_old_usage(0); - size_t old_usage_now = old_gen->used(); - size_t promoted_bytes = old_usage_now - old_usage_before_evac; - heap->set_previous_promotion(promoted_bytes); - - young_gen->unadjust_available(); - old_gen->unadjust_available(); - // No need to old_gen->increase_used(). That was done when plabs were allocated, accounting for both old evacs and promotions. - - young_available = young_gen->adjusted_available(); - old_available = old_gen->adjusted_available(); - - heap->set_alloc_supplement_reserve(0); - heap->set_young_evac_reserve(0); - heap->set_old_evac_reserve(0); - heap->reset_old_evac_expended(); - heap->set_promotion_reserve(0); + if (heap->mode()->is_generational()) { + size_t old_available, young_available; + { + ShenandoahYoungGeneration* young_gen = heap->young_generation(); + ShenandoahGeneration* old_gen = heap->old_generation(); + ShenandoahHeapLocker locker(heap->lock()); + + size_t old_usage_before_evac = heap->capture_old_usage(0); + size_t old_usage_now = old_gen->used(); + size_t promoted_bytes = old_usage_now - old_usage_before_evac; + heap->set_previous_promotion(promoted_bytes); + + young_gen->unadjust_available(); + old_gen->unadjust_available(); + // No need to old_gen->increase_used(). + // That was done when plabs were allocated, accounting for both old evacs and promotions. + + young_available = young_gen->adjusted_available(); + old_available = old_gen->adjusted_available(); + + heap->set_alloc_supplement_reserve(0); + heap->set_young_evac_reserve(0); + heap->set_old_evac_reserve(0); + heap->reset_old_evac_expended(); + heap->set_promotion_reserve(0); + } + log_info(gc, ergo)("At end of concurrent GC, old_available: " SIZE_FORMAT "%s, young_available: " SIZE_FORMAT "%s", + byte_size_in_proper_unit(old_available), proper_unit_for_byte_size(old_available), + byte_size_in_proper_unit(young_available), proper_unit_for_byte_size(young_available)); } - log_info(gc, ergo)("At end of concurrent GC, old_available: " SIZE_FORMAT "%s, young_available: " SIZE_FORMAT "%s", - byte_size_in_proper_unit(old_available), proper_unit_for_byte_size(old_available), - byte_size_in_proper_unit(young_available), proper_unit_for_byte_size(young_available)); return true; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp index ec44434148219..8ce4e2374ab7a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp @@ -53,6 +53,14 @@ ShenandoahDegenGC::ShenandoahDegenGC(ShenandoahDegenPoint degen_point, Shenandoa bool ShenandoahDegenGC::collect(GCCause::Cause cause) { vmop_degenerated(); + ShenandoahHeap* heap = ShenandoahHeap::heap(); + if (heap->mode()->is_generational()) { + size_t old_available = heap->old_generation()->available(); + size_t young_available = heap->young_generation()->available(); + log_info(gc, ergo)("At end of Degenerated GC, old_available: " SIZE_FORMAT "%s, young_available: " SIZE_FORMAT "%s", + byte_size_in_proper_unit(old_available), proper_unit_for_byte_size(old_available), + byte_size_in_proper_unit(young_available), proper_unit_for_byte_size(young_available)); + } return true; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index eca957f1f1805..429410c763086 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -167,7 +167,13 @@ void ShenandoahFullGC::op_full(GCCause::Cause cause) { do_it(cause); metrics.snap_after(); - + if (heap->mode()->is_generational()) { + size_t old_available = heap->old_generation()->available(); + size_t young_available = heap->young_generation()->available(); + log_info(gc, ergo)("At end of Full GC, old_available: " SIZE_FORMAT "%s, young_available: " SIZE_FORMAT "%s", + byte_size_in_proper_unit(old_available), proper_unit_for_byte_size(old_available), + byte_size_in_proper_unit(young_available), proper_unit_for_byte_size(young_available)); + } if (metrics.is_good_progress()) { ShenandoahHeap::heap()->notify_gc_progress(); } else { @@ -182,18 +188,18 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) { // Since we may arrive here from degenerated GC failure of either young or old, establish generation as GLOBAL. heap->set_gc_generation(heap->global_generation()); - // There will be no concurrent allocations during full GC so reset these coordination variables. - heap->young_generation()->unadjust_available(); - heap->old_generation()->unadjust_available(); - // No need to old_gen->increase_used(). That was done when plabs were allocated, accounting for both old evacs and promotions. + if (heap->mode()->is_generational()) { + // There will be no concurrent allocations during full GC so reset these coordination variables. + heap->young_generation()->unadjust_available(); + heap->old_generation()->unadjust_available(); + // No need to old_gen->increase_used(). That was done when plabs were allocated, accounting for both old evacs and promotions. - heap->set_alloc_supplement_reserve(0); - heap->set_young_evac_reserve(0); - heap->set_old_evac_reserve(0); - heap->reset_old_evac_expended(); - heap->set_promotion_reserve(0); + heap->set_alloc_supplement_reserve(0); + heap->set_young_evac_reserve(0); + heap->set_old_evac_reserve(0); + heap->reset_old_evac_expended(); + heap->set_promotion_reserve(0); - if (heap->mode()->is_generational()) { // Full GC supersedes any marking or coalescing in old generation. heap->cancel_old_gc(); } @@ -514,7 +520,7 @@ class ShenandoahPrepareForGenerationalCompactionObjectClosure : public ObjectClo bool promote_object = false; if ((_from_affiliation == ShenandoahRegionAffiliation::YOUNG_GENERATION) && - (from_region_age + object_age > InitialTenuringThreshold)) { + (from_region_age + object_age >= InitialTenuringThreshold)) { if ((_old_to_region != nullptr) && (_old_compact_point + obj_size > _old_to_region->end())) { finish_old_region(); _old_to_region = nullptr; @@ -823,10 +829,12 @@ class ShenandoahEnsureHeapActiveClosure: public ShenandoahHeapRegionClosure { public: ShenandoahEnsureHeapActiveClosure() : _heap(ShenandoahHeap::heap()) {} void heap_region_do(ShenandoahHeapRegion* r) { + bool is_generational = _heap->mode()->is_generational(); if (r->is_trash()) { r->recycle(); } if (r->is_cset()) { + // Leave afffiliation unchanged. r->make_regular_bypass(); } if (r->is_empty_uncommitted()) { @@ -1246,6 +1254,7 @@ class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure { void heap_region_do(ShenandoahHeapRegion* r) { assert (!r->is_cset(), "cset regions should have been demoted already"); + bool is_generational = _heap->mode()->is_generational(); // Need to reset the complete-top-at-mark-start pointer here because // the complete marking bitmap is no longer valid. This ensures @@ -1260,6 +1269,10 @@ class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure { // Make empty regions that have been allocated into regular if (r->is_empty() && live > 0) { + if (!is_generational) { + r->make_young_maybe(); + } + // else, generational mode compaction has already established affiliation. r->make_regular_bypass(); } @@ -1275,7 +1288,7 @@ class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure { } // Update final usage for generations - if (_heap->mode()->is_generational() && live != 0) { + if (is_generational && live != 0) { if (r->is_young()) { _heap->young_generation()->increase_used(live); } else if (r->is_old()) { @@ -1334,6 +1347,7 @@ void ShenandoahFullGC::compact_humongous_objects() { ShenandoahRegionAffiliation original_affiliation = r->affiliation(); for (size_t c = old_start; c <= old_end; c++) { ShenandoahHeapRegion* r = heap->get_region(c); + // Leave humongous region affiliation unchanged. r->make_regular_bypass(); r->set_top(r->bottom()); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index 0ecd4b6bc7502..3d3ff8212cca0 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -110,6 +110,26 @@ void ShenandoahHeapRegion::make_regular_allocation(ShenandoahRegionAffiliation a } } +// Change affiliation to YOUNG_GENERATION if _state is not _pinned_cset, _regular, or _pinned. This implements +// behavior previously performed as a side effect of make_regular_bypass(). +void ShenandoahHeapRegion::make_young_maybe() { + switch (_state) { + case _empty_uncommitted: + case _empty_committed: + case _cset: + case _humongous_start: + case _humongous_cont: + set_affiliation(YOUNG_GENERATION); + return; + case _pinned_cset: + case _regular: + case _pinned: + return; + default: + assert(false, "Unexpected _state in make_young_maybe"); + } +} + void ShenandoahHeapRegion::make_regular_bypass() { shenandoah_assert_heaplocked(); assert (ShenandoahHeap::heap()->is_full_gc_in_progress() || ShenandoahHeap::heap()->is_degenerated_gc_in_progress(), @@ -122,12 +142,6 @@ void ShenandoahHeapRegion::make_regular_bypass() { case _cset: case _humongous_start: case _humongous_cont: - // TODO: Changing this region to young during compaction may not be - // technically correct here because it completely disregards the ages - // and origins of the objects being moved. It is, however, certainly - // more correct than putting live objects into a region without a - // generational affiliation. - set_affiliation(YOUNG_GENERATION); set_state(_regular); return; case _pinned_cset: diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp index 336aff6f43983..36e39c33f4ed5 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp @@ -169,6 +169,7 @@ class ShenandoahHeapRegion { // Allowed transitions from the outside code: void make_regular_allocation(ShenandoahRegionAffiliation affiliation); + void make_young_maybe(); void make_regular_bypass(); void make_humongous_start(); void make_humongous_cont(); From a5aa41308961c28eb4534191f6346d78ff27b290 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Fri, 17 Jun 2022 01:11:11 +0000 Subject: [PATCH 134/254] Improve humongous remset scan Reviewed-by: wkemper --- .../share/gc/shenandoah/shenandoahHeap.cpp | 1 + .../gc/shenandoah/shenandoahHeapRegion.cpp | 50 +++++++++++++++++++ .../gc/shenandoah/shenandoahHeapRegion.hpp | 7 +++ 3 files changed, 58 insertions(+) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 1910001a7b18c..e7003221c6f91 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -2461,6 +2461,7 @@ class ShenandoahUpdateHeapRefsTask : public WorkerTask { // old-gen region in the most recent collection set, or if this card holds pointers to other non-specific // old-gen heap regions. if (r->is_humongous()) { + // Need to examine both dirty and clean cards during mixed evac. r->oop_iterate_humongous(&cl); } else { // This is a mixed evacuation. Old regions that are candidates for collection have not been coalesced diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index 3d3ff8212cca0..19b0317a89157 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -580,6 +580,56 @@ void ShenandoahHeapRegion::global_oop_iterate_objects_and_fill_dead(OopIterateCl } } +// DO NOT CANCEL. If this worker thread has accepted responsibility for scanning a particular range of addresses, it +// must finish the work before it can be cancelled. +void ShenandoahHeapRegion::oop_iterate_humongous_slice(OopIterateClosure* blk, bool dirty_only, + HeapWord* start, size_t words, bool write_table, bool is_concurrent) { + assert(words % CardTable::card_size_in_words() == 0, "Humongous iteration must span whole number of cards"); + assert(is_humongous(), "only humongous region here"); + ShenandoahHeap* heap = ShenandoahHeap::heap(); + + // Find head. + ShenandoahHeapRegion* r = humongous_start_region(); + assert(r->is_humongous_start(), "need humongous head here"); + assert(CardTable::card_size_in_words() * (words / CardTable::card_size_in_words()) == words, + "slice must be integral number of cards"); + + oop obj = cast_to_oop(r->bottom()); + RememberedScanner* scanner = ShenandoahHeap::heap()->card_scan(); + size_t card_index = scanner->card_index_for_addr(start); + size_t num_cards = words / CardTable::card_size_in_words(); + + if (dirty_only) { + if (write_table) { + while (num_cards-- > 0) { + if (scanner->is_write_card_dirty(card_index++)) { + obj->oop_iterate(blk, MemRegion(start, start + CardTable::card_size_in_words())); + } + start += CardTable::card_size_in_words(); + } + } else { + while (num_cards-- > 0) { + if (scanner->is_card_dirty(card_index++)) { + obj->oop_iterate(blk, MemRegion(start, start + CardTable::card_size_in_words())); + } + start += CardTable::card_size_in_words(); + } + } + } else { + // Scan all data, regardless of whether cards are dirty + obj->oop_iterate(blk, MemRegion(start, start + num_cards * CardTable::card_size_in_words())); + } +} + +void ShenandoahHeapRegion::oop_iterate_humongous(OopIterateClosure* blk, HeapWord* start, size_t words) { + assert(is_humongous(), "only humongous region here"); + // Find head. + ShenandoahHeapRegion* r = humongous_start_region(); + assert(r->is_humongous_start(), "need humongous head here"); + oop obj = cast_to_oop(r->bottom()); + obj->oop_iterate(blk, MemRegion(start, start + words)); +} + void ShenandoahHeapRegion::oop_iterate_humongous(OopIterateClosure* blk) { assert(is_humongous(), "only humongous region here"); // Find head. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp index 36e39c33f4ed5..9c3690d7a10af 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp @@ -402,6 +402,13 @@ class ShenandoahHeapRegion { // that are subsumed into coalesced ranges of dead memory need to be "unregistered". void global_oop_iterate_and_fill_dead(OopIterateClosure* cl); void oop_iterate_humongous(OopIterateClosure* cl); + void oop_iterate_humongous(OopIterateClosure* cl, HeapWord* start, size_t words); + + // Invoke closure on every reference contained within the humongous object that spans this humongous + // region if the reference is contained within a DIRTY card and the reference is no more than words following + // start within the humongous object. + void oop_iterate_humongous_slice(OopIterateClosure* cl, bool dirty_only, HeapWord* start, size_t words, + bool write_table, bool is_concurrent); HeapWord* block_start(const void* p) const; size_t block_size(const HeapWord* p) const; From bd13ba314482edf2be66ef7c4a2f7f5f60f3e93d Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Fri, 24 Jun 2022 19:16:19 +0000 Subject: [PATCH 135/254] Pack old evacuations tightly Reviewed-by: wkemper --- .../shenandoahAdaptiveHeuristics.cpp | 21 +- .../heuristics/shenandoahHeuristics.cpp | 40 +- .../heuristics/shenandoahHeuristics.hpp | 2 + .../heuristics/shenandoahOldHeuristics.cpp | 2 +- .../gc/shenandoah/shenandoahCollectionSet.cpp | 7 + .../gc/shenandoah/shenandoahCollectionSet.hpp | 15 + .../shenandoahCollectionSet.inline.hpp | 8 + .../gc/shenandoah/shenandoahConcurrentGC.cpp | 6 +- .../gc/shenandoah/shenandoahDegeneratedGC.cpp | 3 +- .../share/gc/shenandoah/shenandoahFullGC.cpp | 2 +- .../gc/shenandoah/shenandoahGeneration.cpp | 401 ++++++++++++++---- .../share/gc/shenandoah/shenandoahHeap.cpp | 229 ++++++---- .../share/gc/shenandoah/shenandoahHeap.hpp | 20 +- .../gc/shenandoah/shenandoahHeap.inline.hpp | 153 +++++-- .../shenandoah/shenandoahThreadLocalData.hpp | 34 +- .../gc/shenandoah/shenandoah_globals.hpp | 7 + 16 files changed, 730 insertions(+), 220 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp index 53667a39f44d0..7bf052484b6d8 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp @@ -85,8 +85,9 @@ void ShenandoahAdaptiveHeuristics::choose_collection_set_from_regiondata(Shenand // we hit max_cset. When max_cset is hit, we terminate the cset selection. Note that in this scheme, // ShenandoahGarbageThreshold is the soft threshold which would be ignored until min_garbage is hit. - size_t max_cset = (ShenandoahHeap::heap()->get_young_evac_reserve() / ShenandoahEvacWaste); - size_t capacity = ShenandoahHeap::heap()->young_generation()->soft_max_capacity(); + ShenandoahHeap* heap = ShenandoahHeap::heap(); + size_t max_cset = (size_t) (heap->get_young_evac_reserve() / ShenandoahEvacWaste); + size_t capacity = heap->young_generation()->soft_max_capacity(); // As currently implemented, we are not enforcing that new_garbage > min_garbage // size_t free_target = (capacity / 100) * ShenandoahMinFreeThreshold + max_cset; @@ -106,13 +107,17 @@ void ShenandoahAdaptiveHeuristics::choose_collection_set_from_regiondata(Shenand // particular, regions that have reached tenure age will be sorted into this array before younger regions that contain // more garbage. This represents one of the reasons why we keep looking at regions even after we decide, for example, // to exclude one of the regions because it might require evacuation of too much live data. - + bool is_generational = heap->mode()->is_generational(); for (size_t idx = 0; idx < size; idx++) { ShenandoahHeapRegion* r = data[idx]._region; - size_t biased_garbage = data[idx]._garbage; - - size_t new_cset = cur_cset + r->get_live_data_bytes(); - + size_t new_cset; + if (is_generational && (r->age() >= InitialTenuringThreshold)) { + // Entire region will be promoted, This region does not impact young-gen evacuation reserve. Memory has already + // been set aside to hold evacuation results as advance_promotion_reserve. + new_cset = cur_cset; + } else { + new_cset = cur_cset + r->get_live_data_bytes(); + } // As currently implemented, we are not enforcing that new_garbage > min_garbage // size_t new_garbage = cur_garbage + r->garbage(); @@ -139,8 +144,6 @@ void ShenandoahAdaptiveHeuristics::choose_collection_set_from_regiondata(Shenand cset->add_region(r); cur_cset = new_cset; // cur_garbage = new_garbage; - } else if (biased_garbage == 0) { - break; } } } diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp index a87d1c04785af..2448f8d6d716e 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp @@ -77,8 +77,25 @@ ShenandoahHeuristics::~ShenandoahHeuristics() { FREE_C_HEAP_ARRAY(RegionGarbage, _region_data); } -void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collection_set, ShenandoahOldHeuristics* old_heuristics) { +size_t ShenandoahHeuristics::select_aged_regions(size_t old_available, size_t num_regions, bool preselected_regions[]) { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + size_t old_consumed = 0; + if (heap->mode()->is_generational()) { + for (size_t i = 0; i < num_regions; i++) { + ShenandoahHeapRegion* region = heap->get_region(i); + if (in_generation(region) && !region->is_empty() && region->is_regular() && (region->age() >= InitialTenuringThreshold)) { + size_t promotion_need = (size_t) (region->get_live_data_bytes() * ShenandoahEvacWaste); + if (old_consumed + promotion_need < old_available) { + old_consumed += promotion_need; + preselected_regions[i] = true; + } + } + } + } + return old_consumed; +} +void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collection_set, ShenandoahOldHeuristics* old_heuristics) { ShenandoahHeap* heap = ShenandoahHeap::heap(); assert(collection_set->count() == 0, "Must be empty"); @@ -131,9 +148,16 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec live_memory += region->get_live_data_bytes(); // This is our candidate for later consideration. candidates[cand_idx]._region = region; - if (heap->mode()->is_generational() && (region->age() >= InitialTenuringThreshold)) { - // Bias selection of regions that have reached tenure age - for (uint j = region->age() - InitialTenuringThreshold; j > 0; j--) { + if (collection_set->is_preselected(i)) { + // If regions is presected, we know mode()->is_generational() and region->age() >= InitialTenuringThreshold) + + // TODO: Deprecate and/or refine ShenandoahTenuredRegionUsageBias. If we preselect the regions, we can just + // set garbage to "max" value, which is the region size rather than doing this extra work to bias selection. + // May also want to exercise more discretion in select_aged_regions() if we decide there are good reasons + // to not promote all eligible aged regions on the current GC pass. + + // If we're at tenure age, bias at least once. + for (uint j = region->age() + 1 - InitialTenuringThreshold; j > 0; j--) { garbage = (garbage + ShenandoahTenuredRegionUsageBias) * ShenandoahTenuredRegionUsageBias; } } @@ -250,7 +274,7 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec const size_t available_young_regions = free_regions + immediate_regions + young_generation->free_unaffiliated_regions(); const size_t available_old_regions = old_generation->free_unaffiliated_regions(); - size_t already_reserved_old_bytes = heap->get_old_evac_reserve() + heap->get_promotion_reserve(); + size_t already_reserved_old_bytes = heap->get_old_evac_reserve() + heap->get_promoted_reserve(); size_t regions_reserved_for_evac_and_promotion = (already_reserved_old_bytes + region_size_bytes - 1) / region_size_bytes; regions_available_to_loan = available_old_regions - regions_reserved_for_evac_and_promotion; @@ -314,7 +338,7 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec heap->set_alloc_supplement_reserve(potential_evac_supplement); - size_t promotion_budget = heap->get_promotion_reserve(); + size_t promotion_budget = heap->get_promoted_reserve(); size_t old_evac_budget = heap->get_old_evac_reserve(); size_t alloc_budget_evac_and_update = potential_evac_supplement + young_available; @@ -359,6 +383,10 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec byte_size_in_proper_unit(collection_set->garbage()), proper_unit_for_byte_size(collection_set->garbage()), cset_percent); + + size_t bytes_evacuated = collection_set->get_bytes_reserved_for_evacuation(); + log_info(gc, ergo)("Total Evacuation: " SIZE_FORMAT "%s", + byte_size_in_proper_unit(bytes_evacuated), proper_unit_for_byte_size(bytes_evacuated)); } void ShenandoahHeuristics::record_cycle_start() { diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp index a3f088863665d..950e2e987f737 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp @@ -153,6 +153,8 @@ class ShenandoahHeuristics : public CHeapObj { virtual void record_requested_gc(); + virtual size_t select_aged_regions(size_t old_available, size_t num_regions, bool preselected_regions[]); + virtual void choose_collection_set(ShenandoahCollectionSet* collection_set, ShenandoahOldHeuristics* old_heuristics); virtual bool can_unload_classes(); diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp index 160830659af14..1ac6c804a2e53 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp @@ -89,7 +89,7 @@ bool ShenandoahOldHeuristics::prime_collection_set(ShenandoahCollectionSet* coll // promotion until a subsequent evacuation pass. Enforcement is provided at the time PLABs and shared allocations // in old-gen memory are requested. - const size_t promotion_budget_bytes = heap->get_promotion_reserve(); + const size_t promotion_budget_bytes = heap->get_promoted_reserve(); // old_evacuation_budget is an upper bound on the amount of live memory that can be evacuated. // diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp index 2cabf498d8a3d..d23586b64f2f7 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp @@ -44,6 +44,7 @@ ShenandoahCollectionSet::ShenandoahCollectionSet(ShenandoahHeap* heap, ReservedS _garbage(0), _used(0), _region_count(0), + _old_garbage(0), _current_index(0) { // The collection set map is reserved to cover the entire heap *and* zero addresses. @@ -91,9 +92,13 @@ void ShenandoahCollectionSet::add_region(ShenandoahHeapRegion* r) { if (r->affiliation() == YOUNG_GENERATION) { _young_region_count++; _young_bytes_to_evacuate += r->get_live_data_bytes(); + if (r->age() >= InitialTenuringThreshold) { + _young_bytes_to_promote += r->get_live_data_bytes(); + } } else if (r->affiliation() == OLD_GENERATION) { _old_region_count++; _old_bytes_to_evacuate += r->get_live_data_bytes(); + _old_garbage += r->garbage(); } _region_count++; @@ -115,6 +120,7 @@ void ShenandoahCollectionSet::clear() { #endif _garbage = 0; + _old_garbage = 0; _used = 0; _region_count = 0; @@ -123,6 +129,7 @@ void ShenandoahCollectionSet::clear() { _young_region_count = 0; _old_region_count = 0; _young_bytes_to_evacuate = 0; + _young_bytes_to_promote = 0; _old_bytes_to_evacuate = 0; _has_old_regions = false; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp index 6c3398c5499c1..db8f70254f5c1 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp @@ -52,11 +52,18 @@ class ShenandoahCollectionSet : public CHeapObj { // not include bytes reserved for old-generation replicas. The value is // conservative in that memory may be reserved for objects that will be promoted. size_t _young_bytes_to_evacuate; + size_t _young_bytes_to_promote; size_t _old_bytes_to_evacuate; size_t _young_region_count; size_t _old_region_count; + size_t _old_garbage; // How many bytes of old garbage are present in a mixed collection set? + + bool* _preselected_regions; // Points to array identifying which tenure-age regions have been preselected + // for inclusion in collection set. This field is only valid during brief + // spans of time while collection set is being constructed. + shenandoah_padding(0); volatile size_t _current_index; shenandoah_padding(1); @@ -102,10 +109,18 @@ class ShenandoahCollectionSet : public CHeapObj { inline size_t get_old_bytes_reserved_for_evacuation(); inline void reserve_old_bytes_for_evacuation(size_t byte_count); + inline size_t get_young_bytes_to_be_promoted(); + inline size_t get_old_region_count(); inline size_t get_young_region_count(); + inline size_t get_old_garbage(); + + void establish_preselected(bool *preselected) { _preselected_regions = preselected; } + void abandon_preselected() { _preselected_regions = nullptr; } + bool is_preselected(int region_idx) { return (_preselected_regions != nullptr) && _preselected_regions[region_idx]; } + bool has_old_regions() const { return _has_old_regions; } size_t used() const { return _used; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.inline.hpp index 21bda091ea306..5f877af615cad 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.inline.hpp @@ -69,6 +69,10 @@ size_t ShenandoahCollectionSet::get_young_bytes_reserved_for_evacuation() { return _young_bytes_to_evacuate; } +size_t ShenandoahCollectionSet::get_young_bytes_to_be_promoted() { + return _young_bytes_to_promote; +} + size_t ShenandoahCollectionSet::get_bytes_reserved_for_evacuation() { return _young_bytes_to_evacuate + _old_bytes_to_evacuate; } @@ -81,4 +85,8 @@ size_t ShenandoahCollectionSet::get_young_region_count() { return _young_region_count; } +size_t ShenandoahCollectionSet::get_old_garbage() { + return _old_garbage; +} + #endif // SHARE_GC_SHENANDOAH_SHENANDOAHCOLLECTIONSET_INLINE_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index 998cf82c0f6c4..14d847284c6ef 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -241,9 +241,9 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) { heap->set_young_evac_reserve(0); heap->set_old_evac_reserve(0); heap->reset_old_evac_expended(); - heap->set_promotion_reserve(0); + heap->set_promoted_reserve(0); } - log_info(gc, ergo)("At end of concurrent GC, old_available: " SIZE_FORMAT "%s, young_available: " SIZE_FORMAT "%s", + log_info(gc, ergo)("At end of Concurrent GC, old_available: " SIZE_FORMAT "%s, young_available: " SIZE_FORMAT "%s", byte_size_in_proper_unit(old_available), proper_unit_for_byte_size(old_available), byte_size_in_proper_unit(young_available), proper_unit_for_byte_size(young_available)); } @@ -704,7 +704,7 @@ void ShenandoahConcurrentGC::op_final_mark() { // Upon return from prepare_regions_and_collection_set(), certain parameters have been established to govern the // evacuation efforts that are about to begin. In particular: // - // heap->get_promotion_reserve() represents the amount of memory within old-gen's available memory that has + // heap->get_promoted_reserve() represents the amount of memory within old-gen's available memory that has // been set aside to hold objects promoted from young-gen memory. This represents an estimated percentage // of the live young-gen memory within the collection set. If there is more data ready to be promoted than // can fit within this reserve, the promotion of some objects will be deferred until a subsequent evacuation diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp index 8ce4e2374ab7a..e9af9f585a3b5 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp @@ -294,8 +294,7 @@ void ShenandoahDegenGC::op_degenerated() { heap->set_young_evac_reserve(0); heap->set_old_evac_reserve(0); heap->reset_old_evac_expended(); - heap->set_promotion_reserve(0); - + heap->set_promoted_reserve(0); } if (ShenandoahVerify) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index 429410c763086..327d63420b093 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -198,7 +198,7 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) { heap->set_young_evac_reserve(0); heap->set_old_evac_reserve(0); heap->reset_old_evac_expended(); - heap->set_promotion_reserve(0); + heap->set_promoted_reserve(0); // Full GC supersedes any marking or coalescing in old generation. heap->cancel_old_gc(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index 50e4b4f8c92ce..48b1788049d06 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -222,8 +222,10 @@ void ShenandoahGeneration::prepare_gc(bool do_old_gc_bootstrap) { } void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) { - ShenandoahHeap* heap = ShenandoahHeap::heap(); + ShenandoahCollectionSet* collection_set = heap->collection_set(); + size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); + assert(!heap->is_full_gc_in_progress(), "Only for concurrent and degenerated GC"); assert(generation_mode() != OLD, "Only YOUNG and GLOBAL GC perform evacuations"); { @@ -247,109 +249,352 @@ void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) ShenandoahHeapLocker locker(heap->lock()); heap->collection_set()->clear(); - + size_t minimum_evacuation_reserve = ShenandoahOldCompactionReserve * region_size_bytes; + size_t avail_evac_reserve_for_loan_to_young_gen = 0; + size_t old_regions_loaned_for_young_evac = 0; + size_t regions_available_to_loan = 0; + size_t old_evacuation_reserve = 0; + size_t num_regions = heap->num_regions(); + size_t consumed_by_advance_promotion = 0; + bool preselected_regions[num_regions]; + for (unsigned int i = 0; i < num_regions; i++) { + preselected_regions[i] = false; + } if (heap->mode()->is_generational()) { + ShenandoahGeneration* old_generation = heap->old_generation(); + ShenandoahYoungGeneration* young_generation = heap->young_generation(); // During initialization and phase changes, it is more likely that fewer objects die young and old-gen - // memory is not yet full (or is in the process of being replaced). During these tiems especially, it + // memory is not yet full (or is in the process of being replaced). During these times especially, it // is beneficial to loan memory from old-gen to young-gen during the evacuation and update-refs phases // of execution. - // PromotionReserve for old generation: how much memory are we reserving to hold the results of - // promoting young-gen objects that have reached tenure age? This value is not "critical". If we - // underestimate, certain promotions will simply be deferred. The basis of this estimate is - // historical precedent. Conservatively, budget this value to be twice the amount of memory - // promoted in previous GC pass. Whenever the amount promoted during previous GC is zero, - // including initial passes before any objects have reached tenure age, use live memory within - // young-gen memory divided by (ShenandoahTenureAge multiplied by InitialTenuringThreshold) as the - // the very conservative value of this parameter. Note that during initialization, there is - // typically plentiful old-gen memory so it's ok to be conservative with the initial estimates - // of this value. But PromotionReserve can be no larger than available memory. In summary, we - // compute PromotionReserve as the smaller of: - // 1. old_gen->available - // 2. young_gen->capacity() * ShenandoahEvacReserve - // 3. (bytes promoted by previous promotion) * 2 if (bytes promoted by previous promotion) is not zero - // 4. if (bytes promoted by previous promotion) is zero, divide young_gen->used() - // by (ShenandoahTenureAge * InitialTenuringThreshold) + // Calculate EvacuationReserve before PromotionReserve. Evacuation is more critical than promotion. + // If we cannot evacuate old-gen, we will not be able to reclaim old-gen memory. Promotions are less + // critical. If we cannot promote, there may be degradation of young-gen memory because old objects + // accumulate there until they can be promoted. This increases the young-gen marking and evacuation work. + + // Do not fill up old-gen memory with promotions. Reserve some amount of memory for compaction purposes. + ShenandoahOldHeuristics* old_heuristics = heap->old_heuristics(); + if (old_heuristics->unprocessed_old_collection_candidates() > 0) { + + // Compute old_evacuation_reserve: how much memory are we reserving to hold the results of + // evacuating old-gen heap regions? In order to sustain a consistent pace of young-gen collections, + // the goal is to maintain a consistent value for this parameter (when the candidate set is not + // empty). This value is the minimum of: + // 1. old_gen->available() + // 2. old-gen->capacity() * ShenandoahOldEvacReserve) / 100 + // (e.g. old evacuation should be no larger than 5% of old_gen capacity) + // 3. ((young_gen->capacity * ShenandoahEvacReserve / 100) * ShenandoahOldEvacRatioPercent) / 100 + // (e.g. old evacuation should be no larger than 12% of young-gen evacuation) + + old_evacuation_reserve = old_generation->available(); + assert(old_evacuation_reserve > minimum_evacuation_reserve, "Old-gen available has not been preserved!"); + size_t old_evac_reserve_max = old_generation->soft_max_capacity() * ShenandoahOldEvacReserve / 100; + if (old_evac_reserve_max < old_evacuation_reserve) { + old_evacuation_reserve = old_evac_reserve_max; + } + size_t young_evac_reserve_max = + (((young_generation->soft_max_capacity() * ShenandoahEvacReserve) / 100) * ShenandoahOldEvacRatioPercent) / 100; + if (young_evac_reserve_max < old_evacuation_reserve) { + old_evacuation_reserve = young_evac_reserve_max; + } + } + + if (minimum_evacuation_reserve > old_generation->available()) { + // Due to round-off errors during enforcement of minimum_evacuation_reserve during previous GC passes, + // there can be slight discrepancies here. + minimum_evacuation_reserve = old_generation->available(); + } + if (old_evacuation_reserve < minimum_evacuation_reserve) { + // Even if there's nothing to be evacuated on this cycle, we still need to reserve this memory for future + // evacuations. It is ok to loan this memory to young-gen if we don't need it for evacuation on this pass. + avail_evac_reserve_for_loan_to_young_gen = minimum_evacuation_reserve - old_evacuation_reserve; + old_evacuation_reserve = minimum_evacuation_reserve; + } + + heap->set_old_evac_reserve(old_evacuation_reserve); + heap->reset_old_evac_expended(); + + // Compute the young evauation reserve: This is how much memory is available for evacuating young-gen objects. + // We ignore the possible effect of promotions, which reduce demand for young-gen evacuation memory. + // + // TODO: We could give special treatment to the regions that have reached promotion age, because we know their + // live data is entirely eligible for promotion. This knowledge can feed both into calculations of young-gen + // evacuation reserve and promotion reserve. + // + // young_evacuation_reserve for young generation: how much memory are we reserving to hold the results + // of evacuating young collection set regions? This is typically smaller than the total amount + // of available memory, and is also smaller than the total amount of marked live memory within + // young-gen. This value is the smaller of + // + // 1. (young_gen->capacity() * ShenandoahEvacReserve) / 100 + // 2. (young_gen->available() + old_gen_memory_available_to_be_loaned + // + // ShenandoahEvacReserve represents the configured taget size of the evacuation region. We can only honor + // this target if there is memory available to hold the evacuations. Memory is available if it is already + // free within young gen, or if it can be borrowed from old gen. Since we have not yet chosen the collection + // sets, we do not yet know the exact accounting of how many regions will be freed by this collection pass. + // What we do know is that there will be at least one evacuated young-gen region for each old-gen region that + // is loaned to the evacuation effort (because regions to be collected consume more memory than the compacted + // regions that will replace them). In summary, if there are old-gen regions that are available to hold the + // results of young-gen evacuations, it is safe to loan them for this purpose. At this point, we have not yet + // established a promoted_reserve. We'll do that after we choose the collection set and analyze its impact + // on available memory. // - // We don't yet know how much live memory. Inside choose_collection_set(), after it computes live memory, - // the PromotionReserve may be further reduced. + // We do not know the evacuation_supplement until after we have computed the collection set. It is not always + // the case that young-regions inserted into the collection set will result in net decrease of in-use regions + // because ShenandoahEvacWaste times multiplied by memory within the region may be larger than the region size. + // The problem is especially relevant to regions that have been inserted into the collection set because they have + // reached tenure age. These regions tend to have much higher utilization (e.g. 95%). These regions also offer + // a unique opportunity because we know that every live object contained within the region is elgible to be + // promoted. Thus, the following implementation treats these regions specially: // - // 5. live bytes in young-gen divided by (ShenandoahTenureAge * InitialTenuringThreshold - // if the number of bytes promoted by previous promotion is zero + // 1. Before beginning collection set selection, we tally the total amount of live memory held within regions + // that are known to have reached tenure age. If this memory times ShenandoahEvacWaste is available within + // old-gen memory, establish an advance promotion reserve to hold all or some percentage of these objects. + // This advance promotion reserve is excluded from memory available for holding old-gen evacuations and cannot + // be "loaned" to young gen. // + // 2. Tenure-aged regions are included in the collection set iff their evacuation size * ShenandoahEvacWaste fits + // within the advance promotion reserve. It is counter productive to evacuate these regions if they cannot be + // evacuated directly into old-gen memory. So if there is not sufficient memory to hold copies of their + // live data right now, we'll just let these regions remain in young for now, to be evacuated by a subsequent + // evacuation pass. + // + // 3. Next, we calculate a young-gen evacuation budget, which is the smaller of the two quantities mentioned + // above. old_gen_memory_available_to_be_loaned is calculated as: + // old_gen->available - (advance-promotion-reserve + old-gen_evacuation_reserve) + // + // 4. When choosing the collection set, special care is taken to assure that the amount of loaned memory required to + // hold the results of evacuation is smaller than the total memory occupied by the regions added to the collection + // set. We need to take these precautions because we do not know how much memory will be reclaimed by evacuation + // until after the collection set has been constructed. The algorithm is as follows: + // + // a. We feed into the algorithm (i) young available at the start of evacuation and (ii) the amount of memory + // loaned from old-gen that is available to hold the results of evacuation. + // b. As candidate regions are added into the young-gen collection set, we maintain accumulations of the amount + // of memory spanned by the collection set regions and the amount of memory that must be reserved to hold + // evacuation results (by multiplying live-data size by ShenandoahEvacWaste). We process candidate regions + // in order of decreasing amounts of garbage. We skip over (and do not include into the collection set) any + // regions that do not satisfy all of the following conditions: + // + // i. The amount of live data within the region as scaled by ShenandoahEvacWaste must fit within the + // relevant evacuation reserve (live data of old-gen regions must fit within the old-evac-reserve, live + // data of young-gen tenure-aged regions must fit within the advance promotion reserve, live data within + // other young-gen regions must fit within the youn-gen evacuation reserve). + // ii. The accumulation of memory consumed by evacuation must not exceed the accumulation of memory reclaimed + // through evacuation by more than young-gen available. + // iii. Other conditions may be enforced as appropriate for specific heuristics. + // + // Note that regions are considered for inclusion in the selection set in order of decreasing amounts of garbage. + // It is possible that a region with a larger amount of garbage will be rejected because it also has a larger + // amount of live data and some region that follows this region in candidate order is included in the collection + // set (because it has less live data and thus can fit within the evacuation limits even though it has less + // garbage). + + size_t young_evacuation_reserve = (young_generation->max_capacity() * ShenandoahEvacReserve) / 100; + // old evacuation can pack into existing partially used regions. young evacuation and loans for young allocations + // need to target regions that do not already hold any old-gen objects. Round down. + regions_available_to_loan = old_generation->free_unaffiliated_regions(); + consumed_by_advance_promotion = _heuristics->select_aged_regions(old_generation->available() - old_evacuation_reserve, + num_regions, preselected_regions); + size_t net_available_old_regions = + (old_generation->available() - old_evacuation_reserve - consumed_by_advance_promotion) / region_size_bytes; + + if (regions_available_to_loan > net_available_old_regions) { + regions_available_to_loan = net_available_old_regions; + } + // Otherwise, regions_available_to_loan is less than net_available_old_regions because available memory is + // scattered between multiple partially used regions. + + if (young_evacuation_reserve > young_generation->available()) { + size_t short_fall = young_evacuation_reserve - young_generation->available(); + if (regions_available_to_loan * region_size_bytes >= short_fall) { + old_regions_loaned_for_young_evac = (short_fall + region_size_bytes - 1) / region_size_bytes; + regions_available_to_loan -= old_regions_loaned_for_young_evac; + } else { + old_regions_loaned_for_young_evac = regions_available_to_loan; + regions_available_to_loan = 0; + young_evacuation_reserve = young_generation->available() + old_regions_loaned_for_young_evac * region_size_bytes; + } + } else { + old_regions_loaned_for_young_evac = 0; + } + // In generational mode, we may end up choosing a young collection set that contains so many promotable objects + // that there is not sufficient space in old generation to hold the promoted objects. That is ok because we have + // assured there is sufficient space in young generation to hold the rejected promotion candidates. These rejected + // promotion candidates will presumably be promoted in a future evacuation cycle. + heap->set_young_evac_reserve(young_evacuation_reserve); + } else { + // Not generational mode: limit young evac reserve by young available; no need to establish old_evac_reserve. + ShenandoahYoungGeneration* young_generation = heap->young_generation(); + size_t young_evac_reserve = (young_generation->soft_max_capacity() * ShenandoahEvacReserve) / 100; + if (young_evac_reserve > young_generation->available()) { + young_evac_reserve = young_generation->available(); + } + heap->set_young_evac_reserve(young_evac_reserve); + } + + // TODO: young_available can include available (between top() and end()) within each young region that is not + // part of the collection set. Making this memory available to the young_evacuation_reserve allows a larger + // young collection set to be chosen when available memory is under extreme pressure. Implementing this "improvement" + // is tricky, because the incremental construction of the collection set actually changes the amount of memory + // available to hold evacuated young-gen objects. As currently implemented, the memory that is available within + // non-empty regions that are not selected as part of the collection set can be allocated by the mutator while + // GC is evacuating and updating references. + + collection_set->establish_preselected(preselected_regions); + _heuristics->choose_collection_set(heap->collection_set(), heap->old_heuristics()); + collection_set->abandon_preselected(); + + // At this point, young_generation->available() knows about recently discovered immediate garbage. We also + // know the composition of the chosen collection set. + + if (heap->mode()->is_generational()) { ShenandoahGeneration* old_generation = heap->old_generation(); ShenandoahYoungGeneration* young_generation = heap->young_generation(); - size_t promotion_reserve = old_generation->available(); + size_t old_evacuation_committed = (size_t) (ShenandoahEvacWaste * + collection_set->get_old_bytes_reserved_for_evacuation()); + size_t immediate_garbage_regions = collection_set->get_immediate_trash() / region_size_bytes; + + if (old_evacuation_committed > old_evacuation_reserve) { + // This should only happen due to round-off errors when enforcing ShenandoahEvacWaste + assert(old_evacuation_committed < (33 * old_evacuation_reserve) / 32, "Round-off errors should be less than 3.125%%"); + old_evacuation_committed = old_evacuation_reserve; + } - size_t max_young_evacuation = (young_generation->soft_max_capacity() * ShenandoahOldEvacReserve) / 100; - if (max_young_evacuation < promotion_reserve) { - promotion_reserve = max_young_evacuation; + // Recompute old_regions_loaned_for_young_evac because young-gen collection set may not need all the memory + // originally reserved. + + size_t young_evacuation_reserve_used = + collection_set->get_young_bytes_reserved_for_evacuation() - collection_set->get_young_bytes_to_be_promoted(); + young_evacuation_reserve_used = (size_t) (ShenandoahEvacWaste * young_evacuation_reserve_used); + heap->set_young_evac_reserve(young_evacuation_reserve_used); + + // Adjust old_regions_loaned_for_young_evac to feed into calculations of promoted_reserve + if (young_evacuation_reserve_used > young_generation->available()) { + size_t short_fall = young_evacuation_reserve_used - young_generation->available(); + + // region_size_bytes is a power of 2. loan an integral number of regions. + size_t revised_loan_for_young_evacuation = (short_fall + region_size_bytes - 1) / region_size_bytes; + + // Undo the previous loan + regions_available_to_loan += old_regions_loaned_for_young_evac; + old_regions_loaned_for_young_evac = revised_loan_for_young_evacuation; + // And make a new loan + assert(regions_available_to_loan > old_regions_loaned_for_young_evac, "Cannot loan regions that we do not have"); + regions_available_to_loan -= old_regions_loaned_for_young_evac; + } else { + // Undo the prevous loan + regions_available_to_loan += old_regions_loaned_for_young_evac; + old_regions_loaned_for_young_evac = 0; } - size_t previously_promoted = heap->get_previous_promotion(); - if (previously_promoted == 0) { - // Very conservatively, assume linear population decay (rather than more typical exponential) and assume all of - // used is live. - size_t proposed_reserve = young_generation->used() / (ShenandoahAgingCyclePeriod * InitialTenuringThreshold); - if (promotion_reserve > proposed_reserve) { - promotion_reserve = proposed_reserve; - } - } else if (previously_promoted * 2 < promotion_reserve) { - promotion_reserve = previously_promoted * 2; + size_t old_bytes_loaned = old_regions_loaned_for_young_evac * region_size_bytes; + // Need to enforce that old_evacuation_committed + old_bytes_loaned >= minimum_evacuation_reserve + // in order to prevent promotion reserve from violating minimum evacuation reserve. + if (old_evacuation_committed + old_bytes_loaned < minimum_evacuation_reserve) { + // Pretend the old_evacuation_commitment is larger than what will be evacuated to assure that promotions + // do not fill the minimum_evacuation_reserve. Note that regions loaned from old-gen will be returned + // to old-gen before we start a subsequent evacuation. + old_evacuation_committed = minimum_evacuation_reserve - old_bytes_loaned; } - heap->set_promotion_reserve(promotion_reserve); - heap->capture_old_usage(old_generation->used()); + // Limit promoted_reserve so that we can set aside memory to be loaned from old-gen to young-gen. This + // value is not "critical". If we underestimate, certain promotions will simply be deferred. If we put + // "all the rest" of old-gen memory into the promotion reserve, we'll have nothing left to loan to young-gen + // during the evac and update phases of GC. So we "limit" the sizes of the promotion budget to be the smaller of: + // + // 1. old_gen->available - (old_evacuation_committed + old_bytes_loaned + consumed_by_advance_promotion) + // 2. young bytes reserved for evacuation - // OldEvacuationReserve for old generation: how much memory are we reserving to hold the results of - // evacuating old-gen heap regions? In order to sustain a consistent pace of young-gen collections, - // the goal is to maintain a consistent value for this parameter (when the candidate set is not - // empty). This value is the minimum of: - // 1. old_gen->available() - PromotionReserve - // 2. (young_gen->capacity() scaled by ShenandoahEvacReserve) scaled by ShenandoahOldEvacRatioPercent + assert(old_generation->available() > old_evacuation_committed, "Cannot evacuate more than available"); + assert(old_generation->available() > old_evacuation_committed + old_bytes_loaned, "Cannot loan more than available"); + assert(old_generation->available() > old_evacuation_committed + old_bytes_loaned + consumed_by_advance_promotion, + "Cannot promote more than available"); - // Don't reserve for old_evac any more than the memory that is available in old_gen. - size_t old_evacuation_reserve = old_generation->available() - promotion_reserve; + size_t old_avail = old_generation->available(); + size_t promotion_reserve = old_avail - (old_evacuation_committed + consumed_by_advance_promotion + old_bytes_loaned); - // Make sure old evacuation is no more than ShenandoahOldEvacRatioPercent of the total evacuation budget. - size_t max_total_evac = (young_generation->soft_max_capacity() * ShenandoahEvacReserve) / 100; - size_t max_old_evac_portion = (max_total_evac * ShenandoahOldEvacRatioPercent) / 100; + // We experimented with constraining promoted_reserve to be no larger than 4 times the size of previously_promoted, + // but this constraint was too limiting, resulting in failure of legitimate promotions. - if (old_evacuation_reserve > max_old_evac_portion) { - old_evacuation_reserve = max_old_evac_portion; + // We had also experimented with constraining promoted_reserve to be no more than young_evacuation_committed + // divided by promotion_divisor, where: + // size_t promotion_divisor = (0x02 << InitialTenuringThreshold) - 1; + // This also was found to be too limiting, resulting in failure of legitimate promotions. + // + // Both experiments were conducted in the presence of other bugs which could have been the root cause for + // the failures identified above as being "too limiting". TODO: conduct new experiments with the more limiting + // values of young_evacuation_reserved_used. + young_evacuation_reserve_used -= consumed_by_advance_promotion; + if (young_evacuation_reserve_used < promotion_reserve) { + // Shrink promotion_reserve if its larger than the memory to be consumed by evacuating all young objects in + // collection set, including anticipated waste. There's no benefit in using a larger promotion_reserve. + promotion_reserve = young_evacuation_reserve_used; } - heap->set_old_evac_reserve(old_evacuation_reserve); - heap->reset_old_evac_expended(); - - // Compute YoungEvacuationReserve after we prime the collection set with old-gen candidates. This depends - // on how much memory old-gen wants to evacuate. This is done within _heuristics->choose_collection_set(). + assert(old_avail >= promotion_reserve + old_evacuation_committed + old_bytes_loaned + consumed_by_advance_promotion, + "Budget exceeds available old-gen memory"); + log_info(gc, ergo)("Old available: " SIZE_FORMAT ", Original promotion reserve: " SIZE_FORMAT ", Old evacuation reserve: " + SIZE_FORMAT ", Advance promotion reserve supplement: " SIZE_FORMAT ", Old loaned to young: " SIZE_FORMAT, + old_avail, promotion_reserve, old_evacuation_committed, consumed_by_advance_promotion, + old_regions_loaned_for_young_evac * region_size_bytes); + promotion_reserve += consumed_by_advance_promotion; + heap->set_promoted_reserve(promotion_reserve); + heap->reset_promoted_expended(); + if (collection_set->get_old_bytes_reserved_for_evacuation() == 0) { + // Setting old evacuation reserve to zero denotes that there is no old-gen evacuation in this pass. + heap->set_old_evac_reserve(0); + } - // There's no need to pass this information to ShenandoahFreeSet::rebuild(). The GC allocator automatically borrows - // memory from mutator regions when necessary. - } + size_t old_gen_usage_base = old_generation->used() - collection_set->get_old_garbage(); + heap->capture_old_usage(old_gen_usage_base); - // The heuristics may consult and/or change the values of PromotionReserved, OldEvacuationReserved, and - // YoungEvacuationReserved, all of which are represented in the shared ShenandoahHeap data structure. - _heuristics->choose_collection_set(heap->collection_set(), heap->old_heuristics()); + // Compute the evacuation supplement, which is extra memory borrowed from old-gen that can be allocated + // by mutators while GC is working on evacuation and update-refs. This memory can be temporarily borrowed + // from old-gen allotment, then repaid at the end of update-refs from the recycled collection set. After + // we have computed the collection set based on the parameters established above, we can make additional + // loans based on our knowledge of the collection set to determine how much allocation we can allow + // during the evacuation and update-refs phases of execution. The total available supplement is the smaller of: + // + // 1. old_gen->available() - + // (promotion_reserve + old_evacuation_commitment + old_bytes_loaned) + // 2. The replenishment budget (number of regions in collection set - the number of regions already + // under lien for the young_evacuation_reserve) + // - // EvacuationAllocationSupplement: This represents memory that can be allocated in excess of young_gen->available() - // during evacuation and update-refs. This memory can be temporarily borrowed from old-gen allotment, then - // repaid at the end of update-refs from the recycled collection set. After we have computed the collection set - // based on the parameters established above, we can make additional calculates based on our knowledge of the - // collection set to determine how much allocation we can allow during the evacuation and update-refs phases - // of execution. With full awareness of collection set, we can shrink the values of PromotionReserve, - // OldEvacuationReserve, and YoungEvacuationReserve. Then, we can compute EvacuationAllocationReserve as the - // minimum of: - // 1. old_gen->available - (PromotionReserve + OldEvacuationReserve) - // 2. The replenishment budget (number of regions in collection set - the number of regions already - // under lien for the YoungEvacuationReserve) - // - - // The possibly revised values are also consulted by the ShenandoahPacer when it establishes pacing parameters - // for evacuation and update-refs. + size_t young_regions_evacuated = collection_set->get_young_region_count(); + size_t regions_for_runway = 0; + if (young_regions_evacuated > old_regions_loaned_for_young_evac) { + regions_for_runway = young_regions_evacuated - old_regions_loaned_for_young_evac; + old_regions_loaned_for_young_evac = young_regions_evacuated; + regions_available_to_loan -= regions_for_runway; + } + size_t allocation_supplement = regions_for_runway * region_size_bytes; + heap->set_alloc_supplement_reserve(allocation_supplement); + + size_t promotion_budget = heap->get_promoted_reserve(); + size_t old_evac_budget = heap->get_old_evac_reserve(); + size_t alloc_budget_evac_and_update = allocation_supplement + young_generation->available(); + + // TODO: young_available, which feeds into alloc_budget_evac_and_update is lacking memory available within + // existing young-gen regions that were not selected for the collection set. Add this in and adjust the + // log message (where it says "empty-region allocation budget"). + + log_info(gc, ergo)("Memory reserved for evacuation and update-refs includes promotion budget: " SIZE_FORMAT + "%s, young evacuation budget: " SIZE_FORMAT "%s, old evacuation budget: " SIZE_FORMAT + "%s, empty-region allocation budget: " SIZE_FORMAT "%s, including supplement: " SIZE_FORMAT "%s", + byte_size_in_proper_unit(promotion_budget), proper_unit_for_byte_size(promotion_budget), + byte_size_in_proper_unit(young_evacuation_reserve_used), + proper_unit_for_byte_size(young_evacuation_reserve_used), + byte_size_in_proper_unit(old_evac_budget), proper_unit_for_byte_size(old_evac_budget), + byte_size_in_proper_unit(alloc_budget_evac_and_update), + proper_unit_for_byte_size(alloc_budget_evac_and_update), + byte_size_in_proper_unit(allocation_supplement), proper_unit_for_byte_size(allocation_supplement)); + } } { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index e7003221c6f91..1a1f95ed959aa 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -509,7 +509,7 @@ ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) : _regions(NULL), _update_refs_iterator(this), _alloc_supplement_reserve(0), - _promotion_reserve(0), + _promoted_reserve(0), _old_evac_reserve(0), _old_evac_expended(0), _young_evac_reserve(0), @@ -901,82 +901,108 @@ HeapWord* ShenandoahHeap::allocate_from_plab_slow(Thread* thread, size_t size, b size_t min_size = MAX2(size, PLAB::min_size()); // Figure out size of new PLAB, looking back at heuristics. Expand aggressively. - size_t new_size = ShenandoahThreadLocalData::plab_size(thread) * 2; + size_t cur_size = ShenandoahThreadLocalData::plab_size(thread); + if (cur_size == 0) { + cur_size = PLAB::min_size(); + } + size_t future_size = cur_size * 2; // Limit growth of PLABs to ShenandoahMaxEvacLABRatio * the minimum size. This enables more equitable distribution of // available evacuation buidget between the many threads that are coordinating in the evacuation effort. if (ShenandoahMaxEvacLABRatio > 0) { - new_size = MIN2(new_size, PLAB::min_size() * ShenandoahMaxEvacLABRatio); + future_size = MIN2(future_size, PLAB::min_size() * ShenandoahMaxEvacLABRatio); } - new_size = MIN2(new_size, PLAB::max_size()); - new_size = MAX2(new_size, PLAB::min_size()); + future_size = MIN2(future_size, PLAB::max_size()); + future_size = MAX2(future_size, PLAB::min_size()); - size_t unalignment = new_size % CardTable::card_size_in_words(); + size_t unalignment = future_size % CardTable::card_size_in_words(); if (unalignment != 0) { - new_size = new_size - unalignment + CardTable::card_size_in_words(); + future_size = future_size - unalignment + CardTable::card_size_in_words(); } // Record new heuristic value even if we take any shortcut. This captures // the case when moderately-sized objects always take a shortcut. At some point, - // heuristics should catch up with them. Note that the requested new_size may + // heuristics should catch up with them. Note that the requested cur_size may // not be honored, but we remember that this is the preferred size. - ShenandoahThreadLocalData::set_plab_size(thread, new_size); - - if (new_size < size) { - // New size still does not fit the object. Fall back to shared allocation. - // This avoids retiring perfectly good PLABs, when we encounter a large object. - return NULL; + ShenandoahThreadLocalData::set_plab_size(thread, future_size); + if (cur_size < size) { + // The PLAB to be allocated is still not large enough to hold the object. Fall back to shared allocation. + // This avoids retiring perfectly good PLABs in order to represent a single large object allocation. + return nullptr; } // Retire current PLAB, and allocate a new one. PLAB* plab = ShenandoahThreadLocalData::plab(thread); - // CAUTION: retire_plab may register the remnant filler object with the remembered set scanner without a lock. This - // is safe iff it is assured that each PLAB is a whole-number multiple of card-mark memory size and each PLAB is - // aligned with the start of a card's memory range. - retire_plab(plab); - - size_t actual_size = 0; - // allocate_new_plab resets plab_evacuated and plab_promoted and disables promotions if old-gen available is - // less than the remaining evacuation need. - HeapWord* plab_buf = allocate_new_plab(min_size, new_size, &actual_size); - if (plab_buf == NULL) { - return NULL; - } - - assert (size <= actual_size, "allocation should fit"); - - if (ZeroTLAB) { - // ..and clear it. - Copy::zero_to_words(plab_buf, actual_size); - } else { - // ...and zap just allocated object. + if (plab->words_remaining() < PLAB::min_size()) { + // Retire current PLAB, and allocate a new one. + // CAUTION: retire_plab may register the remnant filler object with the remembered set scanner without a lock. This + // is safe iff it is assured that each PLAB is a whole-number multiple of card-mark memory size and each PLAB is + // aligned with the start of a card's memory range. + + retire_plab(plab, thread); + + size_t actual_size = 0; + // allocate_new_plab resets plab_evacuated and plab_promoted and disables promotions if old-gen available is + // less than the remaining evacuation need. It also adjusts plab_preallocated and expend_promoted if appropriate. + HeapWord* plab_buf = allocate_new_plab(min_size, cur_size, &actual_size); + if (plab_buf == NULL) { + return NULL; + } else { + ShenandoahThreadLocalData::enable_plab_retries(thread); + } + assert (size <= actual_size, "allocation should fit"); + if (ZeroTLAB) { + // ..and clear it. + Copy::zero_to_words(plab_buf, actual_size); + } else { + // ...and zap just allocated object. #ifdef ASSERT - // Skip mangling the space corresponding to the object header to - // ensure that the returned space is not considered parsable by - // any concurrent GC thread. - size_t hdr_size = oopDesc::header_size(); - Copy::fill_to_words(plab_buf + hdr_size, actual_size - hdr_size, badHeapWordVal); + // Skip mangling the space corresponding to the object header to + // ensure that the returned space is not considered parsable by + // any concurrent GC thread. + size_t hdr_size = oopDesc::header_size(); + Copy::fill_to_words(plab_buf + hdr_size, actual_size - hdr_size, badHeapWordVal); #endif // ASSERT - } - plab->set_buf(plab_buf, actual_size); + } + plab->set_buf(plab_buf, actual_size); - if (is_promotion && !ShenandoahThreadLocalData::allow_plab_promotions(thread)) { + if (is_promotion && !ShenandoahThreadLocalData::allow_plab_promotions(thread)) { + return nullptr; + } + return plab->allocate(size); + } else { + // If there's still at least min_size() words available within the current plab, don't retire it. Let's gnaw + // away on this plab as long as we can. Meanwhile, return nullptr to force this particular allocation request + // to be satisfied with a shared allocation. By packing more promotions into the previously allocated PLAB, we + // reduce the likelihood of evacuation failures, and we we reduce the need for downsizing our PLABs. return nullptr; } - return plab->allocate(size); } // TODO: It is probably most efficient to register all objects (both promotions and evacuations) that were allocated within // this plab at the time we retire the plab. A tight registration loop will run within both code and data caches. This change // would allow smaller and faster in-line implementation of alloc_from_plab(). Since plabs are aligned on card-table boundaries, // this object registration loop can be performed without acquiring a lock. -void ShenandoahHeap::retire_plab(PLAB* plab) { +void ShenandoahHeap::retire_plab(PLAB* plab, Thread* thread) { if (!mode()->is_generational()) { plab->retire(); } else { - Thread* thread = Thread::current(); - size_t evacuated = ShenandoahThreadLocalData::get_plab_evacuated(thread); - // We don't enforce limits on get_plab_promoted(thread). Promotion uses any memory not required for evacuation. - expend_old_evac(evacuated); + // We don't enforce limits on plab_evacuated. We let it consume all available old-gen memory in order to reduce + // probability of an evacuation failure. We do enforce limits on promotion, to make sure that excessive promotion + // does not result in an old-gen evacuation failure. Note that a failed promotion is relatively harmless. Any + // object that fails to promote in the current cycle will be eligible for promotion in a subsequent cycle. + + // When the plab was instantiated, its entirety was treated as if the entire buffer was going to be dedicated to + // promotions. Now that we are retiring the buffer, we adjust for the reality that the plab is not entirely promotions. + // 1. Some of the plab may have been dedicated to evacuations. + // 2. Some of the plab may have been abandoned due to waste (at the end of the plab). + size_t not_promoted = + ShenandoahThreadLocalData::get_plab_preallocated_promoted(thread) - ShenandoahThreadLocalData::get_plab_promoted(thread); + ShenandoahThreadLocalData::reset_plab_promoted(thread); + ShenandoahThreadLocalData::reset_plab_evacuated(thread); + ShenandoahThreadLocalData::set_plab_preallocated_promoted(thread, 0); + if (not_promoted > 0) { + unexpend_promoted(not_promoted); + } size_t waste = plab->waste(); HeapWord* top = plab->top(); plab->retire(); @@ -991,6 +1017,15 @@ void ShenandoahHeap::retire_plab(PLAB* plab) { } } +void ShenandoahHeap::retire_plab(PLAB* plab) { + if (!mode()->is_generational()) { + plab->retire(); + } else { + Thread* thread = Thread::current(); + retire_plab(plab, thread); + } +} + void ShenandoahHeap::cancel_old_gc() { shenandoah_assert_safepoint(); assert(_old_generation != NULL, "Should only have mixed collections in generation mode."); @@ -1159,9 +1194,14 @@ HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req, bool is_p } HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req, bool& in_new_region, bool is_promotion) { + // promotion_eligible pertains only to PLAB allocations, denoting that the PLAB is allowed to allocate for promotions. + bool promotion_eligible = false; + bool allow_allocation = true; + bool plab_alloc = false; size_t requested_bytes = req.size() * HeapWordSize; - + HeapWord* result = nullptr; ShenandoahHeapLocker locker(lock()); + Thread* thread = Thread::current(); if (mode()->is_generational()) { if (req.affiliation() == YOUNG_GENERATION) { if (req.is_mutator_alloc()) { @@ -1173,32 +1213,35 @@ HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req } } else { // reg.affiliation() == OLD_GENERATION assert(req.type() != ShenandoahAllocRequest::_alloc_gclab, "GCLAB pertains only to young-gen memory"); - if (req.type() == ShenandoahAllocRequest::_alloc_plab) { - // We've already retired this thread's previously exhausted PLAB and have accounted for how that PLAB's - // memory was allotted. - Thread* thread = Thread::current(); - ShenandoahThreadLocalData::reset_plab_evacuated(thread); - ShenandoahThreadLocalData::reset_plab_promoted(thread); - - // Conservatively, assume this entire PLAB will be used for promotion. Act as if we need to serve the - // rest of evacuation need from as-yet unallocated old-gen memory. - size_t remaining_evac_need = get_old_evac_reserve() - get_old_evac_expended(); - size_t evac_available = old_generation()->adjusted_available() - requested_bytes; - if (remaining_evac_need >= evac_available) { - // Disable promotions within this thread because the entirety of this PLAB must be available to hold - // old-gen evacuations. - ShenandoahThreadLocalData::disable_plab_promotions(thread); + plab_alloc = true; + size_t promotion_avail = get_promoted_reserve(); + size_t promotion_expended = get_promoted_expended(); + if (promotion_expended + requested_bytes > promotion_avail) { + promotion_avail = 0; + if (get_old_evac_reserve() == 0) { + // There are no old-gen evacuations in this pass. There's no value in creating a plab that cannot + // be used for promotions. + allow_allocation = false; + } } else { - ShenandoahThreadLocalData::enable_plab_promotions(thread); + promotion_avail = promotion_avail - (promotion_expended + requested_bytes); + promotion_eligible = true; } } else if (is_promotion) { // This is a shared alloc for promotion - Thread* thread = Thread::current(); - size_t remaining_evac_need = get_old_evac_reserve() - get_old_evac_expended(); - size_t evac_available = old_generation()->adjusted_available() - requested_bytes; - if (remaining_evac_need >= evac_available) { - return nullptr; // We need to reserve the remaining memory for evacuation so defer the promotion + size_t promotion_avail = get_promoted_reserve(); + size_t promotion_expended = get_promoted_expended(); + if (promotion_expended + requested_bytes > promotion_avail) { + promotion_avail = 0; + } else { + promotion_avail = promotion_avail - (promotion_expended + requested_bytes); + } + + if (promotion_avail == 0) { + // We need to reserve the remaining memory for evacuation. Reject this allocation. The object will be + // evacuated to young-gen memory and promoted during a future GC pass. + return nullptr; } // Else, we'll allow the allocation to proceed. (Since we hold heap lock, the tested condition remains true.) } else { @@ -1206,10 +1249,32 @@ HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req } } } - - HeapWord* result = _free_set->allocate(req, in_new_region); + result = (allow_allocation)? _free_set->allocate(req, in_new_region): nullptr; if (result != NULL) { if (req.affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) { + ShenandoahThreadLocalData::reset_plab_promoted(thread); + if (req.is_gc_alloc()) { + if (req.type() == ShenandoahAllocRequest::_alloc_plab) { + if (promotion_eligible) { + size_t actual_size = req.actual_size() * HeapWordSize; + // Assume the entirety of this PLAB will be used for promotion. This prevents promotion from overreach. + // When we retire this plab, we'll unexpend what we don't really use. + ShenandoahThreadLocalData::enable_plab_promotions(thread); + expend_promoted(actual_size); + assert(get_promoted_expended() <= get_promoted_reserve(), "Do not expend more promotion than budgeted"); + ShenandoahThreadLocalData::set_plab_preallocated_promoted(thread, actual_size); + } else { + // Disable promotions in this thread because entirety of this PLAB must be available to hold old-gen evacuations. + ShenandoahThreadLocalData::disable_plab_promotions(thread); + ShenandoahThreadLocalData::set_plab_preallocated_promoted(thread, 0); + } + } else if (is_promotion) { + // Shared promotion. Assume size is requested_bytes. + expend_promoted(requested_bytes); + assert(get_promoted_expended() <= get_promoted_reserve(), "Do not expend more promotion than budgeted"); + } + } + // Register the newly allocated object while we're holding the global lock since there's no synchronization // built in to the implementation of register_object(). There are potential races when multiple independent // threads are allocating objects, some of which might span the same card region. For example, consider @@ -1229,6 +1294,14 @@ HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req // invocations to be "mutually exclusive" with respect to each card's memory range. ShenandoahHeap::heap()->card_scan()->register_object(result); } + } else { + // The allocation failed. If this was a plab allocation, We've already retired it and no longer have a plab. + if ((req.affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) && req.is_gc_alloc() && + (req.type() == ShenandoahAllocRequest::_alloc_plab)) { + // We don't need to disable PLAB promotions because there is no PLAB. We leave promotions enabled because + // this allows the surrounding infrastructure to retry alloc_plab_slow() with a smaller PLAB size. + ShenandoahThreadLocalData::set_plab_preallocated_promoted(thread, 0); + } } return result; } @@ -1486,12 +1559,11 @@ class ShenandoahRetireGCLABClosure : public ThreadClosure { PLAB* plab = ShenandoahThreadLocalData::plab(thread); assert(plab != NULL, "PLAB should be initialized for %s", thread->name()); - // TODO; Retiring a PLAB disables it so it cannot support future allocations. This is overkill. For old-gen - // regions, the important thing is to make the memory parsable by the remembered-set scanning code that drives - // the update-refs processing that follows. After the updating of old-gen references is done, it is ok to carve - // this remnant object into smaller pieces during the subsequent evacuation pass, as long as the PLAB is made parsable - // again before the next update-refs phase. - ShenandoahHeap::heap()->retire_plab(plab); + + // There are two reasons to retire all plabs between old-gen evacuation passes. + // 1. We need to make the plab memory parseable by remembered-set scanning. + // 2. We need to establish a trustworthy UpdateWaterMark value within each old-gen heap region + ShenandoahHeap::heap()->retire_plab(plab, thread); if (_resize && ShenandoahThreadLocalData::plab_size(thread) > 0) { ShenandoahThreadLocalData::set_plab_size(thread, 0); } @@ -2428,7 +2500,6 @@ class ShenandoahUpdateHeapRefsTask : public WorkerTask { void do_work(uint worker_id) { T cl; ShenandoahHeapRegion* r = _regions->next(); - // We update references for global, old, and young collections. assert(_heap->active_generation()->is_mark_complete(), "Expected complete marking"); ShenandoahMarkingContext* const ctx = _heap->marking_context(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index 318582d4a1b64..dc5c482b6b8a9 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -350,7 +350,7 @@ class ShenandoahHeap : public CollectedHeap { // // 1. Unadjust the capacity within young-gen and old-gen to undo the effects of borrowing memory from old-gen. Note that // the entirety of the collection set is now available, so allocation capacity naturally increase at this time. - // 2. Clear (reset to zero) _alloc_supplement_reserve, _young_evac_reserve, _old_evac_reserve, and _promotion_reserve + // 2. Clear (reset to zero) _alloc_supplement_reserve, _young_evac_reserve, _old_evac_reserve, and _promoted_reserve // // _young_evac_reserve and _old_evac_reserve are only non-zero during evacuation and update-references. // @@ -364,11 +364,11 @@ class ShenandoahHeap : public CollectedHeap { // Only after "all" threads fail to evacuate an object do we consider the evacuation effort to have failed. intptr_t _alloc_supplement_reserve; // Bytes reserved for young allocations during evac and update refs - size_t _promotion_reserve; // Bytes reserved within old-gen to hold the results of promotion - + size_t _promoted_reserve; // Bytes reserved within old-gen to hold the results of promotion + volatile size_t _promoted_expended; // Bytes of old-gen memory expended on promotions size_t _old_evac_reserve; // Bytes reserved within old-gen to hold evacuated objects from old-gen collection set - size_t _old_evac_expended; // Bytes of old-gen memory expended on old-gen evacuations + volatile size_t _old_evac_expended; // Bytes of old-gen memory expended on old-gen evacuations size_t _young_evac_reserve; // Bytes reserved within young-gen to hold evacuated objects from young-gen collection set @@ -426,8 +426,13 @@ class ShenandoahHeap : public CollectedHeap { inline size_t get_previous_promotion() const; // Returns previous value - inline size_t set_promotion_reserve(size_t new_val); - inline size_t get_promotion_reserve() const; + inline size_t set_promoted_reserve(size_t new_val); + inline size_t get_promoted_reserve() const; + + inline void reset_promoted_expended(); + inline size_t expend_promoted(size_t increment); + inline size_t unexpend_promoted(size_t decrement); + inline size_t get_promoted_expended(); // Returns previous value inline size_t set_old_evac_reserve(size_t new_val); @@ -435,7 +440,7 @@ class ShenandoahHeap : public CollectedHeap { inline void reset_old_evac_expended(); inline size_t expend_old_evac(size_t increment); - inline size_t get_old_evac_expended() const; + inline size_t get_old_evac_expended(); // Returns previous value inline size_t set_young_evac_reserve(size_t new_val); @@ -790,6 +795,7 @@ class ShenandoahHeap : public CollectedHeap { void clear_cards(HeapWord* start, HeapWord* end); void mark_card_as_dirty(void* location); void retire_plab(PLAB* plab); + void retire_plab(PLAB* plab, Thread* thread); void cancel_old_gc(); bool is_old_gc_active(); void coalesce_and_fill_old_regions(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp index b8398d168cddf..d40d8d709d27c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp @@ -301,18 +301,25 @@ inline HeapWord* ShenandoahHeap::allocate_from_plab(Thread* thread, size_t size, assert(UseTLAB, "TLABs should be enabled"); PLAB* plab = ShenandoahThreadLocalData::plab(thread); - if (is_promotion && !ShenandoahThreadLocalData::allow_plab_promotions(thread)) { - return NULL; - } else if (plab == NULL) { - assert(!thread->is_Java_thread() && !thread->is_Worker_thread(), - "Performance: thread should have PLAB: %s", thread->name()); + HeapWord* obj; + if (plab == NULL) { + assert(!thread->is_Java_thread() && !thread->is_Worker_thread(), "Performance: thread should have PLAB: %s", thread->name()); // No PLABs in this thread, fallback to shared allocation - return NULL; + return nullptr; + } else if (is_promotion && (plab->words_remaining() > 0) && !ShenandoahThreadLocalData::allow_plab_promotions(thread)) { + return nullptr; } - HeapWord* obj = plab->allocate(size); - if (obj == NULL) { + // if plab->word_size() <= 0, thread's plab not yet initialized for this pass, so allow_plab_promotions() is not trustworthy + obj = plab->allocate(size); + if ((obj == nullptr) && (plab->words_remaining() < PLAB::min_size())) { + // allocate_from_plab_slow will establish allow_plab_promotions(thread) for future invocations obj = allocate_from_plab_slow(thread, size, is_promotion); } + // if plab->words_remaining() >= PLAB::min_size(), just return nullptr so we can use a shared allocation + if (obj == nullptr) { + return nullptr; + } + if (is_promotion) { ShenandoahThreadLocalData::add_to_plab_promoted(thread, size * HeapWordSize); } else { @@ -350,7 +357,7 @@ inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) { if (result != NULL) { return result; } - // If we failed to promote this aged object, we'll fall through to code below and evacuat to young-gen. + // If we failed to promote this aged object, we'll fall through to code below and evacuate to young-gen. } } return try_evacuate_object(p, thread, r, target_gen); @@ -361,6 +368,7 @@ inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) { inline oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region, ShenandoahRegionAffiliation target_gen) { bool alloc_from_lab = true; + bool has_plab = false; HeapWord* copy = NULL; size_t size = p->size(); bool is_promotion = (target_gen == OLD_GENERATION) && from_region->is_young(); @@ -386,13 +394,30 @@ inline oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, Shenandoah } case OLD_GENERATION: { if (ShenandoahUsePLAB) { + PLAB* plab = ShenandoahThreadLocalData::plab(thread); + if (plab != nullptr) { + has_plab = true; + } copy = allocate_from_plab(thread, size, is_promotion); - if ((copy == nullptr) && (size < ShenandoahThreadLocalData::plab_size(thread))) { - // PLAB allocation failed because we are bumping up against the limit on old evacuation reserve. Try resetting - // the desired PLAB size and retry PLAB allocation to avoid cascading of shared memory allocations. - ShenandoahThreadLocalData::set_plab_size(thread, PLAB::min_size()); - copy = allocate_from_plab(thread, size, is_promotion); - // If we still get nullptr, we'll try a shared allocation below. + if ((copy == nullptr) && (size < ShenandoahThreadLocalData::plab_size(thread)) && + ShenandoahThreadLocalData::plab_retries_enabled(thread)) { + // PLAB allocation failed because we are bumping up against the limit on old evacuation reserve or because + // the requested object does not fit within the current plab but the plab still has an "abundance" of memory, + // where abundance is defined as >= PLAB::min_size(). In the former case, we try resetting the desired + // PLAB size and retry PLAB allocation to avoid cascading of shared memory allocations. + + // In this situation, PLAB memory is precious. We'll try to preserve our existing PLAB by forcing + // this particular allocation to be shared. + if (plab->words_remaining() < PLAB::min_size()) { + ShenandoahThreadLocalData::set_plab_size(thread, PLAB::min_size()); + copy = allocate_from_plab(thread, size, is_promotion); + // If we still get nullptr, we'll try a shared allocation below. + if (copy == nullptr) { + // If retry fails, don't continue to retry until we have success (probably in next GC pass) + ShenandoahThreadLocalData::disable_plab_retries(thread); + } + } + // else, copy still equals nullptr. this causes shared allocation below, preserving this plab for future needs. } } break; @@ -405,10 +430,16 @@ inline oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, Shenandoah } if (copy == NULL) { - // If we failed to allocated in LAB, we'll try a shared allocation. - ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared_gc(size, target_gen); - copy = allocate_memory(req, is_promotion); - alloc_from_lab = false; + // If we failed to allocate in LAB, we'll try a shared allocation. + if (!is_promotion || !has_plab || (size > PLAB::min_size())) { + ShenandoahAllocRequest req = ShenandoahAllocRequest::for_shared_gc(size, target_gen); + copy = allocate_memory(req, is_promotion); + alloc_from_lab = false; + } + // else, we leave copy equal to NULL, signaling a promotion failure below if appropriate. + // We choose not to promote objects smaller than PLAB::min_size() by way of shared allocations, as this is too + // costly. Instead, we'll simply "evacuate" to young-gen memory (using a GCLAB) and will promote in a future + // evacuation pass. This condition is denoted by: is_promotion && has_plab && (size <= PLAB::min_size()) } #ifdef ASSERT } @@ -419,6 +450,55 @@ inline oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, Shenandoah assert(mode()->is_generational(), "Should only be here in generational mode."); if (from_region->is_young()) { // Signal that promotion failed. Will evacuate this old object somewhere in young gen. + + // We squelch excessive reports to reduce noise in logs. Squelch enforcement is not "perfect" because + // this same code can be in-lined in multiple contexts, and each context will have its own copy of the static + // last_report_epoch and this_epoch_report_count variables. + const uint MaxReportsPerEpoch = 4; + static uint last_report_epoch = 0; + static uint epoch_report_count = 0; + PLAB* plab = ShenandoahThreadLocalData::plab(thread); + size_t words_remaining = (plab == nullptr)? 0: plab->words_remaining(); + const char* promote_enabled = ShenandoahThreadLocalData::allow_plab_promotions(thread)? "enabled": "disabled"; + size_t promotion_reserve; + size_t promotion_expended; + // We can only query GCId::current() if current thread is a named thread. If current thread is not a + // named thread, then we don't even try to squelch the promotion failure report, we don't update the + // the last_report_epoch, and we don't increment the epoch_report_count + if (thread->is_Named_thread()) { + uint gc_id = GCId::current(); + if ((gc_id != last_report_epoch) || (epoch_report_count++ < MaxReportsPerEpoch)) { + { + // Promotion failures should be very rare. Invest in providing useful diagnostic info. + ShenandoahHeapLocker locker(lock()); + promotion_reserve = get_promoted_reserve(); + promotion_expended = get_promoted_expended(); + } + log_info(gc, ergo)("Promotion failed, size " SIZE_FORMAT ", has plab? %s, PLAB remaining: " SIZE_FORMAT + ", plab promotions %s, promotion reserve: " SIZE_FORMAT ", promotion expended: " SIZE_FORMAT, + size, plab == nullptr? "no": "yes", + words_remaining, promote_enabled, promotion_reserve, promotion_expended); + if ((gc_id == last_report_epoch) && (epoch_report_count >= MaxReportsPerEpoch)) { + log_info(gc, ergo)("Squelching additional promotion failure reports for epoch %d\n", last_report_epoch); + } else if (gc_id != last_report_epoch) { + last_report_epoch = gc_id;; + epoch_report_count = 1; + } + } + } else if (epoch_report_count < MaxReportsPerEpoch) { + // Unnamed threads are much less common than named threads. In the rare case that an unnamed thread experiences + // a promotion failure before a named thread within a given epoch, the report for the unnamed thread will be squelched. + { + // Promotion failures should be very rare. Invest in providing useful diagnostic info. + ShenandoahHeapLocker locker(lock()); + promotion_reserve = get_promoted_reserve(); + promotion_expended = get_promoted_expended(); + } + log_info(gc, ergo)("Promotion failed (unfiltered), size " SIZE_FORMAT ", has plab? %s, PLAB remaining: " SIZE_FORMAT + ", plab promotions %s, promotion reserve: " SIZE_FORMAT ", promotion expended: " SIZE_FORMAT, + size, plab == nullptr? "no": "yes", + words_remaining, promote_enabled, promotion_reserve, promotion_expended); + } handle_promotion_failure(); return NULL; } else { @@ -588,14 +668,14 @@ inline bool ShenandoahHeap::is_aging_cycle() const { return _is_aging_cycle.is_set(); } -inline size_t ShenandoahHeap::set_promotion_reserve(size_t new_val) { - size_t orig = _promotion_reserve; - _promotion_reserve = new_val; +inline size_t ShenandoahHeap::set_promoted_reserve(size_t new_val) { + size_t orig = _promoted_reserve; + _promoted_reserve = new_val; return orig; } -inline size_t ShenandoahHeap::get_promotion_reserve() const { - return _promotion_reserve; +inline size_t ShenandoahHeap::get_promoted_reserve() const { + return _promoted_reserve; } // returns previous value @@ -624,16 +704,31 @@ inline size_t ShenandoahHeap::get_old_evac_reserve() const { } inline void ShenandoahHeap::reset_old_evac_expended() { - _old_evac_expended = 0; + Atomic::store(&_old_evac_expended, (size_t) 0); } inline size_t ShenandoahHeap::expend_old_evac(size_t increment) { - _old_evac_expended += increment; - return _old_evac_expended; + return Atomic::add(&_old_evac_expended, increment); +} + +inline size_t ShenandoahHeap::get_old_evac_expended() { + return Atomic::load(&_old_evac_expended); +} + +inline void ShenandoahHeap::reset_promoted_expended() { + Atomic::store(&_promoted_expended, (size_t) 0); +} + +inline size_t ShenandoahHeap::expend_promoted(size_t increment) { + return Atomic::add(&_promoted_expended, increment); +} + +inline size_t ShenandoahHeap::unexpend_promoted(size_t decrement) { + return Atomic::sub(&_promoted_expended, decrement); } -inline size_t ShenandoahHeap::get_old_evac_expended() const { - return _old_evac_expended; +inline size_t ShenandoahHeap::get_promoted_expended() { + return Atomic::load(&_promoted_expended); } inline size_t ShenandoahHeap::set_young_evac_reserve(size_t new_val) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp b/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp index b71eb4c740d08..6987ef9228175 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp @@ -56,13 +56,15 @@ class ShenandoahThreadLocalData { PLAB* _plab; size_t _plab_size; - size_t _plab_evacuated; - size_t _plab_promoted; - uint _worker_id; int _disarmed_value; double _paced_time; + size_t _plab_evacuated; + size_t _plab_promoted; + size_t _plab_preallocated_promoted; + bool _plab_retries_enabled; + ShenandoahThreadLocalData() : _gc_state(0), _oom_scope_nesting_level(0), @@ -72,10 +74,12 @@ class ShenandoahThreadLocalData { _gclab_size(0), _plab(NULL), _plab_size(0), + _disarmed_value(0), + _paced_time(0), _plab_evacuated(0), _plab_promoted(0), - _disarmed_value(0), - _paced_time(0) { + _plab_preallocated_promoted(0), + _plab_retries_enabled(true) { // At least on x86_64, nmethod entry barrier encodes _disarmed_value offset // in instruction as disp8 immed @@ -155,6 +159,18 @@ class ShenandoahThreadLocalData { data(thread)->_plab_size = v; } + static void enable_plab_retries(Thread* thread) { + data(thread)->_plab_retries_enabled = true; + } + + static void disable_plab_retries(Thread* thread) { + data(thread)->_plab_retries_enabled = false; + } + + static bool plab_retries_enabled(Thread* thread) { + return data(thread)->_plab_retries_enabled; + } + static void enable_plab_promotions(Thread* thread) { data(thread)->_plab_allows_promotion = true; } @@ -199,6 +215,14 @@ class ShenandoahThreadLocalData { return data(thread)->_plab_promoted; } + static void set_plab_preallocated_promoted(Thread* thread, size_t value) { + data(thread)->_plab_preallocated_promoted = value; + } + + static size_t get_plab_preallocated_promoted(Thread* thread) { + return data(thread)->_plab_preallocated_promoted; + } + static void add_paced_time(Thread* thread, double v) { data(thread)->_paced_time += v; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp index c532f89608243..50b4bd066fa63 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp @@ -521,6 +521,13 @@ "subsequent concurrent mark phase of GC.") \ range(0, 100) \ \ + product(uintx, ShenandoahOldCompactionReserve, 8, EXPERIMENTAL, \ + "During generational GC, prevent promotions from filling " \ + "this number of heap regions. These regions are reserved " \ + "for the purpose of supporting compaction of old-gen " \ + "memory. Otherwise, old-gen memory cannot be compacted.") \ + range(0, 128) \ + \ product(bool, ShenandoahPromoteTenuredObjects, true, DIAGNOSTIC, \ "Turn on/off evacuating individual tenured young objects " \ " to the old generation.") \ From 0fefe704c854f0273935a74b2bb72a50b8a716a4 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Fri, 24 Jun 2022 21:13:49 +0000 Subject: [PATCH 136/254] Remove unnecessary checks identified by reviewer Reviewed-by: wkemper --- .../share/gc/shenandoah/shenandoahHeap.cpp | 66 ++++++++----------- 1 file changed, 29 insertions(+), 37 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 1a1f95ed959aa..13f33a73c360b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -983,47 +983,39 @@ HeapWord* ShenandoahHeap::allocate_from_plab_slow(Thread* thread, size_t size, b // would allow smaller and faster in-line implementation of alloc_from_plab(). Since plabs are aligned on card-table boundaries, // this object registration loop can be performed without acquiring a lock. void ShenandoahHeap::retire_plab(PLAB* plab, Thread* thread) { - if (!mode()->is_generational()) { - plab->retire(); - } else { - // We don't enforce limits on plab_evacuated. We let it consume all available old-gen memory in order to reduce - // probability of an evacuation failure. We do enforce limits on promotion, to make sure that excessive promotion - // does not result in an old-gen evacuation failure. Note that a failed promotion is relatively harmless. Any - // object that fails to promote in the current cycle will be eligible for promotion in a subsequent cycle. - - // When the plab was instantiated, its entirety was treated as if the entire buffer was going to be dedicated to - // promotions. Now that we are retiring the buffer, we adjust for the reality that the plab is not entirely promotions. - // 1. Some of the plab may have been dedicated to evacuations. - // 2. Some of the plab may have been abandoned due to waste (at the end of the plab). - size_t not_promoted = - ShenandoahThreadLocalData::get_plab_preallocated_promoted(thread) - ShenandoahThreadLocalData::get_plab_promoted(thread); - ShenandoahThreadLocalData::reset_plab_promoted(thread); - ShenandoahThreadLocalData::reset_plab_evacuated(thread); - ShenandoahThreadLocalData::set_plab_preallocated_promoted(thread, 0); - if (not_promoted > 0) { - unexpend_promoted(not_promoted); - } - size_t waste = plab->waste(); - HeapWord* top = plab->top(); - plab->retire(); - if (top != NULL && plab->waste() > waste && is_in_old(top)) { - // If retiring the plab created a filler object, then we - // need to register it with our card scanner so it can - // safely walk the region backing the plab. - log_debug(gc)("retire_plab() is registering remnant of size " SIZE_FORMAT " at " PTR_FORMAT, - plab->waste() - waste, p2i(top)); - card_scan()->register_object_wo_lock(top); - } + // We don't enforce limits on plab_evacuated. We let it consume all available old-gen memory in order to reduce + // probability of an evacuation failure. We do enforce limits on promotion, to make sure that excessive promotion + // does not result in an old-gen evacuation failure. Note that a failed promotion is relatively harmless. Any + // object that fails to promote in the current cycle will be eligible for promotion in a subsequent cycle. + + // When the plab was instantiated, its entirety was treated as if the entire buffer was going to be dedicated to + // promotions. Now that we are retiring the buffer, we adjust for the reality that the plab is not entirely promotions. + // 1. Some of the plab may have been dedicated to evacuations. + // 2. Some of the plab may have been abandoned due to waste (at the end of the plab). + size_t not_promoted = + ShenandoahThreadLocalData::get_plab_preallocated_promoted(thread) - ShenandoahThreadLocalData::get_plab_promoted(thread); + ShenandoahThreadLocalData::reset_plab_promoted(thread); + ShenandoahThreadLocalData::reset_plab_evacuated(thread); + ShenandoahThreadLocalData::set_plab_preallocated_promoted(thread, 0); + if (not_promoted > 0) { + unexpend_promoted(not_promoted); + } + size_t waste = plab->waste(); + HeapWord* top = plab->top(); + plab->retire(); + if (top != NULL && plab->waste() > waste && is_in_old(top)) { + // If retiring the plab created a filler object, then we + // need to register it with our card scanner so it can + // safely walk the region backing the plab. + log_debug(gc)("retire_plab() is registering remnant of size " SIZE_FORMAT " at " PTR_FORMAT, + plab->waste() - waste, p2i(top)); + card_scan()->register_object_wo_lock(top); } } void ShenandoahHeap::retire_plab(PLAB* plab) { - if (!mode()->is_generational()) { - plab->retire(); - } else { - Thread* thread = Thread::current(); - retire_plab(plab, thread); - } + Thread* thread = Thread::current(); + retire_plab(plab, thread); } void ShenandoahHeap::cancel_old_gc() { From 3c066d43f2686a160f4a61acd6524b5acf79a8e7 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Tue, 28 Jun 2022 15:53:50 +0000 Subject: [PATCH 137/254] Adaptive heuristic should take reserved memory into account Reviewed-by: kdnilsen --- .../shenandoahAdaptiveHeuristics.cpp | 62 ++++++++++--------- 1 file changed, 34 insertions(+), 28 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp index 7bf052484b6d8..7c91e3116c951 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp @@ -237,13 +237,45 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { size_t soft_tail = max_capacity - capacity; available = (available > soft_tail) ? (available - soft_tail) : 0; + // The collector reserve may eat into what the mutator is allowed to use. Make sure we are looking + // at what is available to the mutator when deciding whether to start a GC. + size_t usable = ShenandoahHeap::heap()->free_set()->available(); + if (usable < available) { + log_debug(gc)("Usable (" SIZE_FORMAT "%s) is less than available (" SIZE_FORMAT "%s)", + byte_size_in_proper_unit(usable), proper_unit_for_byte_size(usable), + byte_size_in_proper_unit(available), proper_unit_for_byte_size(available)); + available = usable; + } + + // Allocation spikes are a characteristic of both the application ahd the JVM configuration. On the JVM command line, + // the application developer may want to supply a hint of the nature of spikes that are inherent in the application + // workload, and this information would normally be independent of heap size (not a percentage thereof). On the + // other hand, some allocation spikes are correlated with JVM configuration. For example, there are allocation + // spikes at the starts of concurrent marking and evacuation to refresh all local allocation buffers. The nature + // of these spikes is determined by LAB min and max sizes and numbers of threads, but also on frequency of GC passes, + // and on "periodic" behavior of these threads If GC frequency is much higher than the periodic trigger for mutator + // threads, then many of the mutator threads may be able to "sit out" of most GC passes. Though the thread's stack + // must be scanned, the thread does not need to refresh its LABs if it sits idle throughout the duration of the GC + // pass. The best prediction for this aspect of spikes in allocation patterns is probably recent past history. + // TODO: and dive deeper into _gc_time_penalties as this may also need to be corrected + + // Check if allocation headroom is still okay. This also factors in: + // 1. Some space to absorb allocation spikes (ShenandoahAllocSpikeFactor) + // 2. Accumulated penalties from Degenerated and Full GC + size_t allocation_headroom = available; + size_t spike_headroom = capacity / 100 * ShenandoahAllocSpikeFactor; + size_t penalties = capacity / 100 * _gc_time_penalties; + + allocation_headroom -= MIN2(allocation_headroom, penalties); + allocation_headroom -= MIN2(allocation_headroom, spike_headroom); + // Track allocation rate even if we decide to start a cycle for other reasons. double rate = _allocation_rate.sample(allocated); _last_trigger = OTHER; size_t min_threshold = capacity / 100 * ShenandoahMinFreeThreshold; - if (available < min_threshold) { + if (allocation_headroom < min_threshold) { log_info(gc)("Trigger (%s): Free (" SIZE_FORMAT "%s) is below minimum threshold (" SIZE_FORMAT "%s)", _generation->name(), byte_size_in_proper_unit(available), proper_unit_for_byte_size(available), @@ -255,7 +287,7 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { const size_t max_learn = ShenandoahLearningSteps; if (_gc_times_learned < max_learn) { size_t init_threshold = capacity / 100 * ShenandoahInitFreeThreshold; - if (available < init_threshold) { + if (allocation_headroom < init_threshold) { log_info(gc)("Trigger (%s): Learning " SIZE_FORMAT " of " SIZE_FORMAT ". Free (" SIZE_FORMAT "%s) is below initial threshold (" SIZE_FORMAT "%s)", _generation->name(), _gc_times_learned + 1, max_learn, byte_size_in_proper_unit(available), proper_unit_for_byte_size(available), @@ -264,27 +296,6 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { } } - // Check if allocation headroom is still okay. This also factors in: - // 1. Some space to absorb allocation spikes - // 2. Accumulated penalties from Degenerated and Full GC - size_t allocation_headroom = available; - - // ShenandoahAllocSpikeFactor is the percentage of capacity that we endeavor to assure to be free at the end of the GC - // cycle. - // TODO: Correct the representation of this quantity - // (and dive deeper into _gc_time_penalties as this may also need to be corrected) - // - // Allocation spikes are a characteristic of both the application ahd the JVM configuration. On the JVM command line, - // the application developer may want to supply a hint of the nature of spikes that are inherent in the application - // workload, and this information would normally be independent of heap size (not a percentage thereof). On the - // other hand, some allocation spikes are correlated with JVM configuration. For example, there are allocation - // spikes at the starts of concurrent marking and evacuation to refresh all local allocation buffers. The nature - // of these spikes is determined by LAB min and max sizes and numbers of threads, but also on frequency of GC passes, - // and on "periodic" behavior of these threads If GC frequency is much higher than the periodic trigger for mutator - // threads, then many of the mutator threads may be able to "sit out" of most GC passes. Though the thread's stack - // must be scanned, the thread does not need to refresh its LABs if it sits idle throughout the duration of the GC - // pass. The best prediction for this aspect of spikes in allocation patterns is probably recent past history. - // // Rationale: // The idea is that there is an average allocation rate and there are occasional abnormal bursts (or spikes) of // allocations that exceed the average allocation rate. What do these spikes look like? @@ -314,8 +325,6 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { // in operation mode. We want some way to decide that the average rate has changed. Make average allocation rate // computations an independent effort. - size_t spike_headroom = capacity / 100 * ShenandoahAllocSpikeFactor; - size_t penalties = capacity / 100 * _gc_time_penalties; // TODO: Account for inherent delays in responding to GC triggers // 1. It has been observed that delays of 200 ms or greater are common between the moment we return true from should_start_gc() @@ -326,9 +335,6 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { // ShenandoahControlIntervalMax. The current control interval (or the max control interval) should also be added into // the calculation of avg_cycle_time below. - allocation_headroom -= MIN2(allocation_headroom, spike_headroom); - allocation_headroom -= MIN2(allocation_headroom, penalties); - double avg_cycle_time = _gc_time_history->davg() + (_margin_of_error_sd * _gc_time_history->dsd()); size_t last_live_memory = get_last_live_memory(); From 4de4c3eb5c3713eb5c06be5247f76df37e0dc043 Mon Sep 17 00:00:00 2001 From: Joshua Cao Date: Wed, 6 Jul 2022 16:22:49 +0000 Subject: [PATCH 138/254] cli options for young/old ShenandoahMinFreeThreshold Reviewed-by: wkemper, ysr --- .../heuristics/shenandoahAdaptiveHeuristics.cpp | 2 +- .../heuristics/shenandoahCompactHeuristics.cpp | 2 +- .../heuristics/shenandoahHeuristics.cpp | 8 ++++++++ .../heuristics/shenandoahHeuristics.hpp | 2 ++ .../heuristics/shenandoahStaticHeuristics.cpp | 2 +- .../share/gc/shenandoah/shenandoah_globals.hpp | 16 ++++++++++++---- 6 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp index 7c91e3116c951..09123165cc72b 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp @@ -273,7 +273,7 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { double rate = _allocation_rate.sample(allocated); _last_trigger = OTHER; - size_t min_threshold = capacity / 100 * ShenandoahMinFreeThreshold; + size_t min_threshold = min_free_threshold(); if (allocation_headroom < min_threshold) { log_info(gc)("Trigger (%s): Free (" SIZE_FORMAT "%s) is below minimum threshold (" SIZE_FORMAT "%s)", diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp index 0591379357f7e..70a3db019be00 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp @@ -55,7 +55,7 @@ bool ShenandoahCompactHeuristics::should_start_gc() { available = (available > soft_tail) ? (available - soft_tail) : 0; size_t threshold_bytes_allocated = capacity / 100 * ShenandoahAllocationThreshold; - size_t min_threshold = capacity / 100 * ShenandoahMinFreeThreshold; + size_t min_threshold = min_free_threshold(); if (available < min_threshold) { log_info(gc)("Trigger: Free (" SIZE_FORMAT "%s) is below minimum threshold (" SIZE_FORMAT "%s)", diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp index 2448f8d6d716e..c8ea5cbe31e34 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp @@ -512,6 +512,14 @@ bool ShenandoahHeuristics::in_generation(ShenandoahHeapRegion* region) { || (_generation->generation_mode() == OLD && region->affiliation() == OLD_GENERATION)); } +size_t ShenandoahHeuristics::min_free_threshold() { + size_t min_free_threshold = + _generation->generation_mode() == GenerationMode::OLD + ? ShenandoahOldMinFreeThreshold + : ShenandoahMinFreeThreshold; + return _generation->soft_max_capacity() / 100 * min_free_threshold; +} + void ShenandoahHeuristics::save_last_live_memory(size_t live_memory) { _live_memory_penultimate_cycle = _live_memory_last_cycle; _live_memory_last_cycle = live_memory; diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp index 950e2e987f737..27c62b73ce366 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp @@ -119,6 +119,8 @@ class ShenandoahHeuristics : public CHeapObj { bool in_generation(ShenandoahHeapRegion* region); + size_t min_free_threshold(); + public: ShenandoahHeuristics(ShenandoahGeneration* generation); virtual ~ShenandoahHeuristics(); diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp index 978b0a8f91797..255cca792edf4 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp @@ -49,7 +49,7 @@ bool ShenandoahStaticHeuristics::should_start_gc() { size_t soft_tail = max_capacity - capacity; available = (available > soft_tail) ? (available - soft_tail) : 0; - size_t threshold_available = capacity / 100 * ShenandoahMinFreeThreshold; + size_t threshold_available = min_free_threshold(); if (available < threshold_available) { log_info(gc)("Trigger: Free (" SIZE_FORMAT "%s) is below minimum threshold (" SIZE_FORMAT "%s)", diff --git a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp index 50b4bd066fa63..e395106bce03b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp @@ -111,10 +111,18 @@ range(0,100) \ \ product(uintx, ShenandoahMinFreeThreshold, 10, EXPERIMENTAL, \ - "Percentage of free heap memory below which most heuristics " \ - "trigger collection independent of other triggers. Provides " \ - "a safety margin for many heuristics. In percents of (soft) " \ - "max heap size.") \ + "Percentage of free heap memory (or young generation, in " \ + "generational mode) below which most heuristics trigger " \ + "collection independent of other triggers. Provides a safety " \ + "margin for many heuristics. In percents of (soft) max heap " \ + "size.") \ + range(0,100) \ + \ + product(uintx, ShenandoahOldMinFreeThreshold, 5, EXPERIMENTAL, \ + "Percentage of free old generation heap memory below which most " \ + "heuristics trigger collection independent of other triggers. " \ + "Provides a safety margin for many heuristics. In percents of " \ + "(soft) max heap size.") \ range(0,100) \ \ product(uintx, ShenandoahAllocationThreshold, 0, EXPERIMENTAL, \ From 09d6ab574ae84ae597948708c31f0d1c5df86830 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Wed, 6 Jul 2022 17:14:01 +0000 Subject: [PATCH 139/254] Windows compiler does not support variable length arrays Reviewed-by: ysr, kdnilsen --- .../share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp | 3 +-- src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp | 2 +- src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp | 6 ++++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp index c8ea5cbe31e34..499e3f7ce1e85 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp @@ -123,7 +123,6 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec ShenandoahMarkingContext* const ctx = _generation->complete_marking_context(); - size_t remnant_available = 0; for (size_t i = 0; i < num_regions; i++) { ShenandoahHeapRegion* region = heap->get_region(i); if (!in_generation(region)) { @@ -149,7 +148,7 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec // This is our candidate for later consideration. candidates[cand_idx]._region = region; if (collection_set->is_preselected(i)) { - // If regions is presected, we know mode()->is_generational() and region->age() >= InitialTenuringThreshold) + // If region is preselected, we know mode()->is_generational() and region->age() >= InitialTenuringThreshold) // TODO: Deprecate and/or refine ShenandoahTenuredRegionUsageBias. If we preselect the regions, we can just // set garbage to "max" value, which is the region size rather than doing this extra work to bias selection. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp index db8f70254f5c1..f42a74024238c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp @@ -119,7 +119,7 @@ class ShenandoahCollectionSet : public CHeapObj { void establish_preselected(bool *preselected) { _preselected_regions = preselected; } void abandon_preselected() { _preselected_regions = nullptr; } - bool is_preselected(int region_idx) { return (_preselected_regions != nullptr) && _preselected_regions[region_idx]; } + bool is_preselected(size_t region_idx) { return (_preselected_regions != nullptr) && _preselected_regions[region_idx]; } bool has_old_regions() const { return _has_old_regions; } size_t used() const { return _used; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index 48b1788049d06..62c1a0ee9467a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -256,10 +256,11 @@ void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) size_t old_evacuation_reserve = 0; size_t num_regions = heap->num_regions(); size_t consumed_by_advance_promotion = 0; - bool preselected_regions[num_regions]; + bool* preselected_regions = NEW_C_HEAP_ARRAY(bool, num_regions, mtGC); for (unsigned int i = 0; i < num_regions; i++) { preselected_regions[i] = false; } + if (heap->mode()->is_generational()) { ShenandoahGeneration* old_generation = heap->old_generation(); ShenandoahYoungGeneration* young_generation = heap->young_generation(); @@ -445,8 +446,9 @@ void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) // GC is evacuating and updating references. collection_set->establish_preselected(preselected_regions); - _heuristics->choose_collection_set(heap->collection_set(), heap->old_heuristics()); + _heuristics->choose_collection_set(collection_set, heap->old_heuristics()); collection_set->abandon_preselected(); + FREE_C_HEAP_ARRAY(bool, preselected_regions); // At this point, young_generation->available() knows about recently discovered immediate garbage. We also // know the composition of the chosen collection set. From 013b2f693834b0e5ed6c6d2d197ebfcf12b63ae5 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Tue, 12 Jul 2022 22:01:18 +0000 Subject: [PATCH 140/254] Include a version identifier in region sampling protocol for visualizer Co-authored-by: Celine Wu Reviewed-by: kdnilsen --- .../share/gc/shenandoah/shenandoahHeapRegionCounters.cpp | 5 ++++- .../share/gc/shenandoah/shenandoahHeapRegionCounters.hpp | 2 ++ .../share/gc/shenandoah/shenandoahLogFileOutput.cpp | 7 ++++--- .../share/gc/shenandoah/shenandoahLogFileOutput.hpp | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp index 687c3ffc8e1ad..30bb63217c040 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp @@ -53,6 +53,9 @@ ShenandoahHeapRegionCounters::ShenandoahHeapRegionCounters() : cname = PerfDataManager::counter_name(_name_space, "max_regions"); PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_None, num_regions, CHECK); + cname = PerfDataManager::counter_name(_name_space, "protocol_version"); //creating new protocol_version + PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_None, VERSION_NUMBER, CHECK); + cname = PerfDataManager::counter_name(_name_space, "region_size"); PerfDataManager::create_constant(SUN_GC, cname, PerfData::U_None, ShenandoahHeapRegion::region_size_bytes() >> 10, CHECK); @@ -116,7 +119,7 @@ void ShenandoahHeapRegionCounters::update() { // If logging enabled, dump current region snapshot to log file if (ShenandoahLogRegionSampling && _log_file != NULL) { - _log_file->write_snapshot(_regions_data, _timestamp, _status, num_regions, rs >> 10); + _log_file->write_snapshot(_regions_data, _timestamp, _status, num_regions, rs >> 10, VERSION_NUMBER); } } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.hpp index 6b02a6e4d97ce..038ee48065f34 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.hpp @@ -82,6 +82,8 @@ class ShenandoahHeapRegionCounters : public CHeapObj { static const jlong AFFILIATION_SHIFT = 56; static const jlong STATUS_SHIFT = 58; + static const jlong VERSION_NUMBER = 2; + char* _name_space; PerfLongVariable** _regions_data; PerfLongVariable* _timestamp; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahLogFileOutput.cpp b/src/hotspot/share/gc/shenandoah/shenandoahLogFileOutput.cpp index 2a2c0e279effe..c73580d4cd1d8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahLogFileOutput.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahLogFileOutput.cpp @@ -103,14 +103,15 @@ int ShenandoahLogFileOutput::write_snapshot(PerfLongVariable** regions, PerfLongVariable* ts, PerfLongVariable* status, size_t num_regions, - size_t rs) { + size_t region_size, size_t protocol_version) { int written = 0; + FileLocker flocker(_stream); - WRITE_LOG_WITH_RESULT_CHECK(jio_fprintf(_stream, "%lli %lli %u %u\n", + WRITE_LOG_WITH_RESULT_CHECK(jio_fprintf(_stream, "%lli %lli %u %u %u\n", ts->get_value(), status->get_value(), num_regions, - rs),written); + region_size, protocol_version), written); if (num_regions > 0) { WRITE_LOG_WITH_RESULT_CHECK(jio_fprintf(_stream, "%lli", regions[0]->get_value()), written); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahLogFileOutput.hpp b/src/hotspot/share/gc/shenandoah/shenandoahLogFileOutput.hpp index 9a3b8a880d563..218a79494cc42 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahLogFileOutput.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahLogFileOutput.hpp @@ -65,7 +65,7 @@ class ShenandoahLogFileOutput : public CHeapObj { PerfLongVariable* ts, PerfLongVariable* status, size_t num_regions, - size_t rs); + size_t region_size, size_t protocolVersion); const char* name() const { return _name; From d47c60ecd81c05bda39191e44ee916a5aab22baa Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Wed, 13 Jul 2022 16:59:38 +0000 Subject: [PATCH 141/254] Refactor budgeting to make the logic cleaner Reviewed-by: rkennke, wkemper --- .../shenandoahAdaptiveHeuristics.cpp | 158 ++-- .../heuristics/shenandoahHeuristics.cpp | 161 +--- .../heuristics/shenandoahOldHeuristics.cpp | 87 +- .../gc/shenandoah/shenandoahConcurrentGC.cpp | 2 +- .../share/gc/shenandoah/shenandoahFreeSet.cpp | 8 +- .../gc/shenandoah/shenandoahGeneration.cpp | 777 ++++++++++-------- .../gc/shenandoah/shenandoahGeneration.hpp | 12 + .../share/gc/shenandoah/shenandoahHeap.cpp | 21 +- .../gc/shenandoah/shenandoahHeap.inline.hpp | 1 + .../shenandoahReferenceProcessor.cpp | 1 - .../gc/shenandoah/shenandoah_globals.hpp | 25 +- 11 files changed, 607 insertions(+), 646 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp index 09123165cc72b..af3bb6e153e29 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp @@ -67,6 +67,8 @@ void ShenandoahAdaptiveHeuristics::choose_collection_set_from_regiondata(Shenand RegionData* data, size_t size, size_t actual_free) { size_t garbage_threshold = ShenandoahHeapRegion::region_size_bytes() * ShenandoahGarbageThreshold / 100; + size_t ignore_threshold = ShenandoahHeapRegion::region_size_bytes() * ShenandoahIgnoreGarbageThreshold / 100; + ShenandoahHeap* heap = ShenandoahHeap::heap(); // The logic for cset selection in adaptive is as follows: // @@ -85,65 +87,117 @@ void ShenandoahAdaptiveHeuristics::choose_collection_set_from_regiondata(Shenand // we hit max_cset. When max_cset is hit, we terminate the cset selection. Note that in this scheme, // ShenandoahGarbageThreshold is the soft threshold which would be ignored until min_garbage is hit. - ShenandoahHeap* heap = ShenandoahHeap::heap(); - size_t max_cset = (size_t) (heap->get_young_evac_reserve() / ShenandoahEvacWaste); - size_t capacity = heap->young_generation()->soft_max_capacity(); - - // As currently implemented, we are not enforcing that new_garbage > min_garbage - // size_t free_target = (capacity / 100) * ShenandoahMinFreeThreshold + max_cset; - // size_t min_garbage = (free_target > actual_free ? (free_target - actual_free) : 0); - - log_info(gc, ergo)("Adaptive CSet Selection. Max CSet: " SIZE_FORMAT "%s, Actual Free: " SIZE_FORMAT "%s.", - byte_size_in_proper_unit(max_cset), proper_unit_for_byte_size(max_cset), - byte_size_in_proper_unit(actual_free), proper_unit_for_byte_size(actual_free)); - - // Better select garbage-first regions - QuickSort::sort(data, (int)size, compare_by_garbage, false); - - size_t cur_cset = 0; - // size_t cur_garbage = 0; - // In generational mode, the sort order within the data array is not strictly descending amounts of garbage. In // particular, regions that have reached tenure age will be sorted into this array before younger regions that contain // more garbage. This represents one of the reasons why we keep looking at regions even after we decide, for example, // to exclude one of the regions because it might require evacuation of too much live data. bool is_generational = heap->mode()->is_generational(); - for (size_t idx = 0; idx < size; idx++) { - ShenandoahHeapRegion* r = data[idx]._region; - size_t new_cset; - if (is_generational && (r->age() >= InitialTenuringThreshold)) { - // Entire region will be promoted, This region does not impact young-gen evacuation reserve. Memory has already - // been set aside to hold evacuation results as advance_promotion_reserve. - new_cset = cur_cset; + bool is_global = (_generation->generation_mode() == GLOBAL); + size_t capacity = heap->young_generation()->max_capacity(); + size_t cur_young_garbage = 0; + + // Better select garbage-first regions + QuickSort::sort(data, (int)size, compare_by_garbage, false); + + if (is_generational) { + if (is_global) { + size_t max_young_cset = (size_t) (heap->get_young_evac_reserve() / ShenandoahEvacWaste); + size_t young_cur_cset = 0; + size_t max_old_cset = (size_t) (heap->get_old_evac_reserve() / ShenandoahEvacWaste); + size_t old_cur_cset = 0; + size_t free_target = (capacity * ShenandoahMinFreeThreshold) / 100 + max_young_cset; + size_t min_garbage = (free_target > actual_free) ? (free_target - actual_free) : 0; + + log_info(gc, ergo)("Adaptive CSet Selection for GLOBAL. Max Young Cset: " SIZE_FORMAT + "%s, Max Old CSet: " SIZE_FORMAT "%s, Actual Free: " SIZE_FORMAT "%s.", + byte_size_in_proper_unit(max_young_cset), proper_unit_for_byte_size(max_young_cset), + byte_size_in_proper_unit(max_old_cset), proper_unit_for_byte_size(max_old_cset), + byte_size_in_proper_unit(actual_free), proper_unit_for_byte_size(actual_free)); + + for (size_t idx = 0; idx < size; idx++) { + ShenandoahHeapRegion* r = data[idx]._region; + bool add_region = false; + if (r->is_old()) { + size_t new_cset = old_cur_cset + r->get_live_data_bytes(); + if ((new_cset <= max_old_cset) && (r->garbage() > garbage_threshold)) { + add_region = true; + old_cur_cset = new_cset; + } + } else if (r->age() >= InitialTenuringThreshold) { + // Entire region will be promoted, This region does not impact young-gen or old-gen evacuation reserve. + // This region has been pre-selected and its impact on promotion reserve is already accounted for. + add_region = true; + cur_young_garbage += r->garbage(); + } else { + size_t new_cset = young_cur_cset + r->get_live_data_bytes(); + size_t region_garbage = r->garbage(); + size_t new_garbage = cur_young_garbage + region_garbage; + bool add_regardless = (region_garbage > ignore_threshold) && (new_garbage < min_garbage); + if ((new_cset <= max_young_cset) && (add_regardless || (region_garbage > garbage_threshold))) { + add_region = true; + young_cur_cset = new_cset; + cur_young_garbage = new_garbage; + } + } + + if (add_region) { + cset->add_region(r); + } + } } else { - new_cset = cur_cset + r->get_live_data_bytes(); + // This is young-gen collection. + size_t max_cset = (size_t) (heap->get_young_evac_reserve() / ShenandoahEvacWaste); + size_t cur_cset = 0; + size_t free_target = (capacity * ShenandoahMinFreeThreshold) / 100 + max_cset; + size_t min_garbage = (free_target > actual_free) ? (free_target - actual_free) : 0; + + log_info(gc, ergo)("Adaptive CSet Selection for YOUNG. Max CSet: " SIZE_FORMAT "%s, Actual Free: " SIZE_FORMAT "%s.", + byte_size_in_proper_unit(max_cset), proper_unit_for_byte_size(max_cset), + byte_size_in_proper_unit(actual_free), proper_unit_for_byte_size(actual_free)); + + for (size_t idx = 0; idx < size; idx++) { + ShenandoahHeapRegion* r = data[idx]._region; + size_t new_cset; + size_t region_garbage = r->garbage(); + size_t new_garbage = cur_young_garbage + region_garbage; + if (r->age() >= InitialTenuringThreshold) { + // Entire region will be promoted, This region does not impact young-gen evacuation reserve. Memory has already + // been set aside to hold evacuation results as advance_promotion_reserve. + new_cset = cur_cset; + } else { + new_cset = cur_cset + r->get_live_data_bytes(); + } + bool add_regardless = (region_garbage > ignore_threshold) && (new_garbage < min_garbage); + if ((new_cset <= max_cset) && + (add_regardless || (region_garbage > garbage_threshold) || (r->age() >= InitialTenuringThreshold))) { + cset->add_region(r); + cur_cset = new_cset; + cur_young_garbage = new_garbage; + } + } } - // As currently implemented, we are not enforcing that new_garbage > min_garbage - // size_t new_garbage = cur_garbage + r->garbage(); - - // Note that live data bytes within a region is not the same as heap_region_size - garbage. This is because - // each region contains a combination of used memory (which is garbage plus live) and unused memory, which has not - // yet been allocated. It may be the case that the region on this iteration has too much live data to be added to - // the collection set while one or more regions seen on subsequent iterations of this loop can be added to the collection - // set because they have smaller live memory, even though they also have smaller garbage (and necessarily a larger - // amount of unallocated memory). - - // BANDAID: In an earlier version of this code, this was written: - // if ((new_cset <= max_cset) && ((new_garbage < min_garbage) || (r->garbage() > garbage_threshold))) - // The problem with the original code is that in some cases the collection set would include hundreds of regions, - // each with less than 100 bytes of garbage. Evacuating these regions is counterproductive. - - // TODO: Think about changing the description and defaults for ShenandoahGarbageThreshold and ShenandoahMinFreeThreshold. - // If "customers" want to evacuate regions with smaller amounts of garbage contained therein, they should specify a lower - // value of ShenandoahGarbageThreshold. As implemented currently, we may experience back-to-back collections if there is - // not enough memory to be reclaimed. Let's not let pursuit of min_garbage drive us to make poor decisions. Maybe we - // want yet another global parameter to allow a region to be placed into the collection set if - // (((new_garbage < min_garbage) && (r->garbage() > ShenandoahSmallerGarbageThreshold)) || (r->garbage() > garbage_threshold)) - - if ((new_cset <= max_cset) && ((r->garbage() > garbage_threshold) || (r->age() >= InitialTenuringThreshold))) { - cset->add_region(r); - cur_cset = new_cset; - // cur_garbage = new_garbage; + } else { + // Traditional Shenandoah (non-generational) + size_t max_cset = (size_t) (heap->get_young_evac_reserve() / ShenandoahEvacWaste); + size_t cur_cset = 0; + size_t free_target = (capacity * ShenandoahMinFreeThreshold) / 100 + max_cset; + size_t min_garbage = (free_target > actual_free) ? (free_target - actual_free) : 0; + + log_info(gc, ergo)("Adaptive CSet Selection. Max CSet: " SIZE_FORMAT "%s, Actual Free: " SIZE_FORMAT "%s.", + byte_size_in_proper_unit(max_cset), proper_unit_for_byte_size(max_cset), + byte_size_in_proper_unit(actual_free), proper_unit_for_byte_size(actual_free)); + + for (size_t idx = 0; idx < size; idx++) { + ShenandoahHeapRegion* r = data[idx]._region; + size_t new_cset = cur_cset + r->get_live_data_bytes(); + size_t region_garbage = r->garbage(); + size_t new_garbage = cur_young_garbage + region_garbage; + bool add_regardless = (region_garbage > ignore_threshold) && (new_garbage < min_garbage); + if ((new_cset <= max_cset) && (add_regardless || (region_garbage > garbage_threshold))) { + cset->add_region(r); + cur_cset = new_cset; + cur_young_garbage = new_garbage; + } } } } diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp index 499e3f7ce1e85..d055124ac06e3 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp @@ -89,6 +89,8 @@ size_t ShenandoahHeuristics::select_aged_regions(size_t old_available, size_t nu old_consumed += promotion_need; preselected_regions[i] = true; } + // Note that we keep going even if one region is excluded from selection. Subsequent regions may be selected + // if they have smaller live data. } } } @@ -97,6 +99,7 @@ size_t ShenandoahHeuristics::select_aged_regions(size_t old_available, size_t nu void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collection_set, ShenandoahOldHeuristics* old_heuristics) { ShenandoahHeap* heap = ShenandoahHeap::heap(); + bool is_generational = heap->mode()->is_generational(); assert(collection_set->count() == 0, "Must be empty"); assert(_generation->generation_mode() != OLD, "Old GC invokes ShenandoahOldHeuristics::choose_collection_set()"); @@ -131,7 +134,6 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec size_t garbage = region->garbage(); total_garbage += garbage; - if (region->is_empty()) { free_regions++; free += ShenandoahHeapRegion::region_size_bytes(); @@ -143,22 +145,12 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec region->make_trash_immediate(); } else { assert (_generation->generation_mode() != OLD, "OLD is handled elsewhere"); - live_memory += region->get_live_data_bytes(); // This is our candidate for later consideration. candidates[cand_idx]._region = region; - if (collection_set->is_preselected(i)) { + if (is_generational && collection_set->is_preselected(i)) { // If region is preselected, we know mode()->is_generational() and region->age() >= InitialTenuringThreshold) - - // TODO: Deprecate and/or refine ShenandoahTenuredRegionUsageBias. If we preselect the regions, we can just - // set garbage to "max" value, which is the region size rather than doing this extra work to bias selection. - // May also want to exercise more discretion in select_aged_regions() if we decide there are good reasons - // to not promote all eligible aged regions on the current GC pass. - - // If we're at tenure age, bias at least once. - for (uint j = region->age() + 1 - InitialTenuringThreshold; j > 0; j--) { - garbage = (garbage + ShenandoahTenuredRegionUsageBias) * ShenandoahTenuredRegionUsageBias; - } + garbage = ShenandoahHeapRegion::region_size_bytes(); } candidates[cand_idx]._garbage = garbage; cand_idx++; @@ -202,158 +194,17 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec byte_size_in_proper_unit(total_garbage), proper_unit_for_byte_size(total_garbage)); size_t immediate_percent = (total_garbage == 0) ? 0 : (immediate_garbage * 100 / total_garbage); + collection_set->set_immediate_trash(immediate_garbage); if (immediate_percent <= ShenandoahImmediateThreshold) { - if (old_heuristics != NULL) { old_heuristics->prime_collection_set(collection_set); - - size_t bytes_reserved_for_old_evacuation = collection_set->get_old_bytes_reserved_for_evacuation(); - if (bytes_reserved_for_old_evacuation * ShenandoahEvacWaste < heap->get_old_evac_reserve()) { - size_t old_evac_reserve = (size_t) (bytes_reserved_for_old_evacuation * ShenandoahEvacWaste); - heap->set_old_evac_reserve(old_evac_reserve); - } } // else, this is global collection and doesn't need to prime_collection_set - ShenandoahYoungGeneration* young_generation = heap->young_generation(); - size_t young_evacuation_reserve = (young_generation->soft_max_capacity() * ShenandoahEvacReserve) / 100; - - // At this point, young_generation->available() does not know about recently discovered immediate garbage. - // What memory it does think to be available is not entirely trustworthy because any available memory associated - // with a region that is placed into the collection set becomes unavailable when the region is chosen - // for the collection set. We'll compute an approximation of young available. If young_available is zero, - // we'll need to borrow from old-gen in order to evacuate. If there's nothing to borrow, we're going to - // degenerate to full GC. - - // TODO: young_available can include available (between top() and end()) within each young region that is not - // part of the collection set. Making this memory available to the young_evacuation_reserve allows a larger - // young collection set to be chosen when available memory is under extreme pressure. Implementing this "improvement" - // is tricky, because the incremental construction of the collection set actually changes the amount of memory - // available to hold evacuated young-gen objects. As currently implemented, the memory that is available within - // non-empty regions that are not selected as part of the collection set can be allocated by the mutator while - // GC is evacuating and updating references. - - size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); - size_t free_affiliated_regions = immediate_regions + free_regions; - size_t young_available = (free_affiliated_regions + young_generation->free_unaffiliated_regions()) * region_size_bytes; - - size_t regions_available_to_loan = 0; - - if (heap->mode()->is_generational()) { - // Now that we've primed the collection set, we can figure out how much memory to reserve for evacuation - // of young-gen objects. - // - // YoungEvacuationReserve for young generation: how much memory are we reserving to hold the results - // of evacuating young collection set regions? This is typically smaller than the total amount - // of available memory, and is also smaller than the total amount of marked live memory within - // young-gen. This value is the minimum of: - // 1. young_gen->available() + (old_gen->available - (OldEvacuationReserve + PromotionReserve)) - // 2. young_gen->capacity() * ShenandoahEvacReserve - // - // Note that any region added to the collection set will be completely evacuated and its memory will - // be completely recycled at the end of GC. The recycled memory will be at least as great as the - // memory borrowed from old-gen. Enforce that the amount borrowed from old-gen for YoungEvacuationReserve - // is an integral number of entire heap regions. - // - young_evacuation_reserve -= heap->get_old_evac_reserve(); - - // Though we cannot know the evacuation_supplement until after we have computed the collection set, we do - // know that every young-gen region added to the collection set will have a net positive impact on available - // memory within young-gen, since each contributes a positive amount of garbage to available. Thus, even - // without knowing the exact composition of the collection set, we can allow young_evacuation_reserve to - // exceed young_available if there are empty regions available within old-gen to hold the results of evacuation. - - ShenandoahGeneration* old_generation = heap->old_generation(); - - // Not all of what is currently available within young-gen can be reserved to hold the results of young-gen - // evacuation. This is because memory available within any heap region that is placed into the collection set - // is not available to be allocated during evacuation. To be safe, we assure that all memory required for evacuation - // is available within "virgin" heap regions. - - const size_t available_young_regions = free_regions + immediate_regions + young_generation->free_unaffiliated_regions(); - const size_t available_old_regions = old_generation->free_unaffiliated_regions(); - size_t already_reserved_old_bytes = heap->get_old_evac_reserve() + heap->get_promoted_reserve(); - size_t regions_reserved_for_evac_and_promotion = (already_reserved_old_bytes + region_size_bytes - 1) / region_size_bytes; - regions_available_to_loan = available_old_regions - regions_reserved_for_evac_and_promotion; - - if (available_young_regions * region_size_bytes < young_evacuation_reserve) { - // Try to borrow old-gen regions in order to avoid shrinking young_evacuation_reserve - size_t loan_request = young_evacuation_reserve - available_young_regions * region_size_bytes; - size_t loaned_region_request = (loan_request + region_size_bytes - 1) / region_size_bytes; - if (loaned_region_request > regions_available_to_loan) { - // Scale back young_evacuation_reserve to consume all available young and old regions. After the - // collection set is chosen, we may get some of this memory back for pacing allocations during evacuation - // and update refs. - loaned_region_request = regions_available_to_loan; - young_evacuation_reserve = (available_young_regions + loaned_region_request) * region_size_bytes; - } else { - // No need to scale back young_evacuation_reserve. - } - } else { - // No need scale back young_evacuation_reserve and no need to borrow from old-gen. We may even have some - // available_young_regions to support allocation pacing. - } - - } else if (young_evacuation_reserve > young_available) { - // In non-generational mode, there's no old-gen memory to borrow from - young_evacuation_reserve = young_available; - } - - heap->set_young_evac_reserve(young_evacuation_reserve); - // Add young-gen regions into the collection set. This is a virtual call, implemented differently by each // of the heuristics subclasses. choose_collection_set_from_regiondata(collection_set, candidates, cand_idx, immediate_garbage + free); - - // Now compute the evacuation supplement, which is extra memory borrowed from old-gen that can be allocated - // by mutators while GC is working on evacuation and update-refs. - - // During evacuation and update refs, we will be able to allocate any memory that is currently available - // plus any memory that can be borrowed on the collateral of the current collection set, reserving a certain - // percentage of the anticipated replenishment from collection set memory to be allocated during the subsequent - // concurrent marking effort. This is how much I can repay. - size_t potential_supplement_regions = collection_set->get_young_region_count(); - - // Though I can repay potential_supplement_regions, I can't borrow them unless they are available in old-gen. - if (potential_supplement_regions > regions_available_to_loan) { - potential_supplement_regions = regions_available_to_loan; - } - - size_t potential_evac_supplement; - - // How much of the potential_supplement_regions will be consumed by young_evacuation_reserve: borrowed_evac_regions. - const size_t available_unaffiliated_young_regions = young_generation->free_unaffiliated_regions(); - const size_t available_affiliated_regions = free_regions + immediate_regions; - const size_t available_young_regions = available_unaffiliated_young_regions + available_affiliated_regions; - size_t young_evac_regions = (young_evacuation_reserve + region_size_bytes - 1) / region_size_bytes; - size_t borrowed_evac_regions = (young_evac_regions > available_young_regions)? young_evac_regions - available_young_regions: 0; - - potential_supplement_regions -= borrowed_evac_regions; - potential_evac_supplement = potential_supplement_regions * region_size_bytes; - - // Leave some allocation runway for subsequent concurrent mark phase. - potential_evac_supplement = (potential_evac_supplement * ShenandoahBorrowPercent) / 100; - - heap->set_alloc_supplement_reserve(potential_evac_supplement); - - size_t promotion_budget = heap->get_promoted_reserve(); - size_t old_evac_budget = heap->get_old_evac_reserve(); - size_t alloc_budget_evac_and_update = potential_evac_supplement + young_available; - - // TODO: young_available, which feeds into alloc_budget_evac_and_update is lacking memory available within - // existing young-gen regions that were not selected for the collection set. Add this in and adjust the - // log message (where it says "empty-region allocation budget"). - - log_info(gc, ergo)("Memory reserved for evacuation and update-refs includes promotion budget: " SIZE_FORMAT - "%s, young evacuation budget: " SIZE_FORMAT "%s, old evacuation budget: " SIZE_FORMAT - "%s, empty-region allocation budget: " SIZE_FORMAT "%s, including supplement: " SIZE_FORMAT "%s", - byte_size_in_proper_unit(promotion_budget), proper_unit_for_byte_size(promotion_budget), - byte_size_in_proper_unit(young_evacuation_reserve), proper_unit_for_byte_size(young_evacuation_reserve), - byte_size_in_proper_unit(old_evac_budget), proper_unit_for_byte_size(old_evac_budget), - byte_size_in_proper_unit(alloc_budget_evac_and_update), - proper_unit_for_byte_size(alloc_budget_evac_and_update), - byte_size_in_proper_unit(potential_evac_supplement), proper_unit_for_byte_size(potential_evac_supplement)); } else { // we're going to skip evacuation and update refs because we reclaimed sufficient amounts of immediate garbage. heap->shenandoah_policy()->record_abbreviated_cycle(); diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp index 1ac6c804a2e53..b3f127bf0400b 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp @@ -51,94 +51,39 @@ bool ShenandoahOldHeuristics::prime_collection_set(ShenandoahCollectionSet* coll ShenandoahHeap* heap = ShenandoahHeap::heap(); uint included_old_regions = 0; size_t evacuated_old_bytes = 0; + size_t collected_old_bytes = 0; - // TODO: - // The max_old_evacuation_bytes and promotion_budget_bytes constants represent a first - // approximation to desired operating parameters. Eventually, these values should be determined - // by heuristics and should adjust dynamically based on most current execution behavior. In the - // interim, we offer command-line options to set the values of these configuration parameters. - - // max_old_evacuation_bytes represents a bound on how much evacuation effort is dedicated - // to old-gen regions. - size_t max_old_evacuation_bytes = (heap->old_generation()->soft_max_capacity() * ShenandoahOldEvacReserve) / 100; - const size_t young_evacuation_bytes = (heap->young_generation()->soft_max_capacity() * ShenandoahEvacReserve) / 100; - const size_t ratio_bound_on_old_evac_bytes = (young_evacuation_bytes * ShenandoahOldEvacRatioPercent) / 100; - if (max_old_evacuation_bytes > ratio_bound_on_old_evac_bytes) { - max_old_evacuation_bytes = ratio_bound_on_old_evac_bytes; - } - - // Usually, old-evacuation is limited by the CPU bounds on effort. However, it can also be bounded by available - // memory within old-gen to hold the results of evacuation. When we are bound by memory availability, we need - // to account below for the loss of available memory from within each region that is added to the old-gen collection - // set. - size_t old_available = heap->old_generation()->available(); - size_t excess_old_capacity_for_evacuation; - if (max_old_evacuation_bytes > old_available) { - max_old_evacuation_bytes = old_available; - excess_old_capacity_for_evacuation = 0; - } else { - excess_old_capacity_for_evacuation = old_available - max_old_evacuation_bytes; - } - - // promotion_budget_bytes represents an "arbitrary" bound on how many bytes can be consumed by young-gen - // objects promoted into old-gen memory. We need to avoid a scenario under which promotion of objects - // depletes old-gen available memory to the point that there is insufficient memory to hold old-gen objects - // that need to be evacuated from within the old-gen collection set. - // - // Key idea: if there is not sufficient memory within old-gen to hold an object that wants to be promoted, defer - // promotion until a subsequent evacuation pass. Enforcement is provided at the time PLABs and shared allocations - // in old-gen memory are requested. - - const size_t promotion_budget_bytes = heap->get_promoted_reserve(); - - // old_evacuation_budget is an upper bound on the amount of live memory that can be evacuated. - // // If a region is put into the collection set, then this region's free (not yet used) bytes are no longer // "available" to hold the results of other evacuations. This may cause a decrease in the remaining amount // of memory that can still be evacuated. We address this by reducing the evacuation budget by the amount // of live memory in that region and by the amount of unallocated memory in that region if the evacuation - // budget is constrained by availability of free memory. See remaining_old_evacuation_budget below. - - size_t old_evacuation_budget = (size_t) (max_old_evacuation_bytes / ShenandoahEvacWaste); - - log_info(gc)("Choose old regions for mixed collection: old evacuation budget: " SIZE_FORMAT "%s", - byte_size_in_proper_unit(old_evacuation_budget), proper_unit_for_byte_size(old_evacuation_budget)); - + // budget is constrained by availability of free memory. + size_t old_evacuation_budget = (size_t) (heap->get_old_evac_reserve() / ShenandoahEvacWaste); size_t remaining_old_evacuation_budget = old_evacuation_budget; size_t lost_evacuation_capacity = 0; + log_info(gc)("Choose old regions for mixed collection: old evacuation budget: " SIZE_FORMAT "%s, candidates: %u", + byte_size_in_proper_unit(old_evacuation_budget), proper_unit_for_byte_size(old_evacuation_budget), + unprocessed_old_collection_candidates()); - // The number of old-gen regions that were selected as candidates for collection at the end of the most recent - // old-gen concurrent marking phase and have not yet been collected is represented by - // unprocessed_old_collection_candidates() + // The number of old-gen regions that were selected as candidates for collection at the end of the most recent old-gen + // concurrent marking phase and have not yet been collected is represented by unprocessed_old_collection_candidates() while (unprocessed_old_collection_candidates() > 0) { // Old collection candidates are sorted in order of decreasing garbage contained therein. ShenandoahHeapRegion* r = next_old_collection_candidate(); - // If we choose region r to be collected, then we need to decrease the capacity to hold other evacuations by // the size of r's free memory. - if ((r->get_live_data_bytes() <= remaining_old_evacuation_budget) && - ((lost_evacuation_capacity + r->free() <= excess_old_capacity_for_evacuation) - || (r->get_live_data_bytes() + r->free() <= remaining_old_evacuation_budget))) { - // Decrement remaining evacuation budget by bytes that will be copied. If the cumulative loss of free memory from - // regions that are to be collected exceeds excess_old_capacity_for_evacuation, decrease - // remaining_old_evacuation_budget by this loss as well. + // It's probably overkill to compensate with lost_evacuation_capacity. But it's the safe thing to do and + // has minimal impact on content of primed collection set. + if (r->get_live_data_bytes() + lost_evacuation_capacity <= remaining_old_evacuation_budget) { + // Decrement remaining evacuation budget by bytes that will be copied. lost_evacuation_capacity += r->free(); remaining_old_evacuation_budget -= r->get_live_data_bytes(); - if (lost_evacuation_capacity > excess_old_capacity_for_evacuation) { - // This is slightly conservative because we really only need to remove from the remaining evacuation budget - // the amount by which lost_evacution_capacity exceeds excess_old_capacity_for_evacuation, but this is relatively - // rare event and current thought is to be a bit conservative rather than mess up the math on code that is so - // difficult to test and maintain... - - // Once we have crossed the threshold of lost_evacuation_capacity exceeding excess_old_capacity_for_evacuation, - // every subsequent iteration of this loop will further decrease remaining_old_evacuation_budget. - remaining_old_evacuation_budget -= r->free(); - } collection_set->add_region(r); included_old_regions++; evacuated_old_bytes += r->get_live_data_bytes(); + collected_old_bytes += r->garbage(); consume_old_collection_candidate(); } else { break; @@ -146,8 +91,10 @@ bool ShenandoahOldHeuristics::prime_collection_set(ShenandoahCollectionSet* coll } if (included_old_regions > 0) { - log_info(gc)("Old-gen piggyback evac (" UINT32_FORMAT " regions, " SIZE_FORMAT " %s)", - included_old_regions, byte_size_in_proper_unit(evacuated_old_bytes), proper_unit_for_byte_size(evacuated_old_bytes)); + log_info(gc)("Old-gen piggyback evac (" UINT32_FORMAT " regions, evacuating " SIZE_FORMAT "%s, reclaiming: " SIZE_FORMAT "%s)", + included_old_regions, + byte_size_in_proper_unit(evacuated_old_bytes), proper_unit_for_byte_size(evacuated_old_bytes), + byte_size_in_proper_unit(collected_old_bytes), proper_unit_for_byte_size(collected_old_bytes)); } return (included_old_regions > 0); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index 14d847284c6ef..54b168937f96e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -754,7 +754,7 @@ void ShenandoahConcurrentGC::op_final_mark() { // loaned for young-gen allocations or evacuations. size_t old_available = heap->old_generation()->adjust_available(-heap->get_alloc_supplement_reserve()); - log_info(gc, ergo)("After generational memory budget adjustments, old avaiable: " SIZE_FORMAT + log_info(gc, ergo)("After generational memory budget adjustments, old available: " SIZE_FORMAT "%s, young_available: " SIZE_FORMAT "%s", byte_size_in_proper_unit(old_available), proper_unit_for_byte_size(old_available), byte_size_in_proper_unit(young_available), proper_unit_for_byte_size(young_available)); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index 37bdfd30032ae..b20b323c91f95 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -468,7 +468,6 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req) { adjust_bounds(); } assert_bounds(); - req.set_actual_size(words_size); return _heap->get_region(beg)->bottom(); } @@ -578,7 +577,12 @@ void ShenandoahFreeSet::rebuild() { reserve_regions(to_reserve); } else { size_t young_reserve = (_heap->young_generation()->max_capacity() / 100) * ShenandoahEvacReserve; - size_t old_reserve = (_heap->old_generation()->max_capacity() / 100) * ShenandoahOldEvacReserve; + // Note that all allocations performed from old-gen are performed by GC, generally using PLABs for both + // promotions and evacuations. The partition between which old memory is reserved for evacuation and + // which is reserved for promotion is enforced using thread-local variables that prescribe intentons within + // each PLAB. We do not reserve any of old-gen memory in order to facilitate the loaning of old-gen memory + // to young-gen purposes. + size_t old_reserve = 0; size_t to_reserve = young_reserve + old_reserve; reserve_regions(to_reserve); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index 62c1a0ee9467a..6630b80226ff1 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -221,7 +221,426 @@ void ShenandoahGeneration::prepare_gc(bool do_old_gc_bootstrap) { } } -void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) { +void ShenandoahGeneration::compute_evacuation_budgets(ShenandoahHeap* heap, bool* preselected_regions, + ShenandoahCollectionSet* collection_set, + size_t &old_regions_loaned_for_young_evac, size_t ®ions_available_to_loan, + size_t &minimum_evacuation_reserve, size_t &consumed_by_advance_promotion) { + size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); + minimum_evacuation_reserve = ShenandoahOldCompactionReserve * region_size_bytes; + old_regions_loaned_for_young_evac = 0; + regions_available_to_loan = 0; + consumed_by_advance_promotion = 0; + if (heap->mode()->is_generational()) { + ShenandoahGeneration* old_generation = heap->old_generation(); + ShenandoahYoungGeneration* young_generation = heap->young_generation(); + size_t avail_evac_reserve_for_loan_to_young_gen = 0; + size_t old_evacuation_reserve = 0; + size_t num_regions = heap->num_regions(); + + // During initialization and phase changes, it is more likely that fewer objects die young and old-gen + // memory is not yet full (or is in the process of being replaced). During these times especially, it + // is beneficial to loan memory from old-gen to young-gen during the evacuation and update-refs phases + // of execution. + + // Calculate EvacuationReserve before PromotionReserve. Evacuation is more critical than promotion. + // If we cannot evacuate old-gen, we will not be able to reclaim old-gen memory. Promotions are less + // critical. If we cannot promote, there may be degradation of young-gen memory because old objects + // accumulate there until they can be promoted. This increases the young-gen marking and evacuation work. + + // Do not fill up old-gen memory with promotions. Reserve some amount of memory for compaction purposes. + ShenandoahOldHeuristics* old_heuristics = heap->old_heuristics(); + size_t young_evac_reserve_max = 0; + if (old_heuristics->unprocessed_old_collection_candidates() > 0) { + // Compute old_evacuation_reserve: how much memory are we reserving to hold the results of + // evacuating old-gen heap regions? In order to sustain a consistent pace of young-gen collections, + // the goal is to maintain a consistent value for this parameter (when the candidate set is not + // empty). This value is the minimum of: + // 1. old_gen->available() + // 2. old-gen->capacity() * ShenandoahOldEvacReserve) / 100 + // (e.g. old evacuation should be no larger than 5% of old_gen capacity) + // 3. ((young_gen->capacity * ShenandoahEvacReserve / 100) * ShenandoahOldEvacRatioPercent) / 100 + // (e.g. old evacuation should be no larger than 12% of young-gen evacuation) + old_evacuation_reserve = old_generation->available(); + assert(old_evacuation_reserve > minimum_evacuation_reserve, "Old-gen available has not been preserved!"); + size_t old_evac_reserve_max = old_generation->soft_max_capacity() * ShenandoahOldEvacReserve / 100; + if (old_evac_reserve_max < old_evacuation_reserve) { + old_evacuation_reserve = old_evac_reserve_max; + } + young_evac_reserve_max = + (((young_generation->soft_max_capacity() * ShenandoahEvacReserve) / 100) * ShenandoahOldEvacRatioPercent) / 100; + if (young_evac_reserve_max < old_evacuation_reserve) { + old_evacuation_reserve = young_evac_reserve_max; + } + } + + if (minimum_evacuation_reserve > old_generation->available()) { + // Due to round-off errors during enforcement of minimum_evacuation_reserve during previous GC passes, + // there can be slight discrepancies here. + minimum_evacuation_reserve = old_generation->available(); + } + if (old_evacuation_reserve < minimum_evacuation_reserve) { + // Even if there's nothing to be evacuated on this cycle, we still need to reserve this memory for future + // evacuations. It is ok to loan this memory to young-gen if we don't need it for evacuation on this pass. + avail_evac_reserve_for_loan_to_young_gen = minimum_evacuation_reserve - old_evacuation_reserve; + old_evacuation_reserve = minimum_evacuation_reserve; + } + + heap->set_old_evac_reserve(old_evacuation_reserve); + heap->reset_old_evac_expended(); + + // Compute the young evauation reserve: This is how much memory is available for evacuating young-gen objects. + // We ignore the possible effect of promotions, which reduce demand for young-gen evacuation memory. + // + // TODO: We could give special treatment to the regions that have reached promotion age, because we know their + // live data is entirely eligible for promotion. This knowledge can feed both into calculations of young-gen + // evacuation reserve and promotion reserve. + // + // young_evacuation_reserve for young generation: how much memory are we reserving to hold the results + // of evacuating young collection set regions? This is typically smaller than the total amount + // of available memory, and is also smaller than the total amount of marked live memory within + // young-gen. This value is the smaller of + // + // 1. (young_gen->capacity() * ShenandoahEvacReserve) / 100 + // 2. (young_gen->available() + old_gen_memory_available_to_be_loaned + // + // ShenandoahEvacReserve represents the configured taget size of the evacuation region. We can only honor + // this target if there is memory available to hold the evacuations. Memory is available if it is already + // free within young gen, or if it can be borrowed from old gen. Since we have not yet chosen the collection + // sets, we do not yet know the exact accounting of how many regions will be freed by this collection pass. + // What we do know is that there will be at least one evacuated young-gen region for each old-gen region that + // is loaned to the evacuation effort (because regions to be collected consume more memory than the compacted + // regions that will replace them). In summary, if there are old-gen regions that are available to hold the + // results of young-gen evacuations, it is safe to loan them for this purpose. At this point, we have not yet + // established a promoted_reserve. We'll do that after we choose the collection set and analyze its impact + // on available memory. + // + // We do not know the evacuation_supplement until after we have computed the collection set. It is not always + // the case that young-regions inserted into the collection set will result in net decrease of in-use regions + // because ShenandoahEvacWaste times multiplied by memory within the region may be larger than the region size. + // The problem is especially relevant to regions that have been inserted into the collection set because they have + // reached tenure age. These regions tend to have much higher utilization (e.g. 95%). These regions also offer + // a unique opportunity because we know that every live object contained within the region is elgible to be + // promoted. Thus, the following implementation treats these regions specially: + // + // 1. Before beginning collection set selection, we tally the total amount of live memory held within regions + // that are known to have reached tenure age. If this memory times ShenandoahEvacWaste is available within + // old-gen memory, establish an advance promotion reserve to hold all or some percentage of these objects. + // This advance promotion reserve is excluded from memory available for holding old-gen evacuations and cannot + // be "loaned" to young gen. + // + // 2. Tenure-aged regions are included in the collection set iff their evacuation size * ShenandoahEvacWaste fits + // within the advance promotion reserve. It is counter productive to evacuate these regions if they cannot be + // evacuated directly into old-gen memory. So if there is not sufficient memory to hold copies of their + // live data right now, we'll just let these regions remain in young for now, to be evacuated by a subsequent + // evacuation pass. + // + // 3. Next, we calculate a young-gen evacuation budget, which is the smaller of the two quantities mentioned + // above. old_gen_memory_available_to_be_loaned is calculated as: + // old_gen->available - (advance-promotion-reserve + old-gen_evacuation_reserve) + // + // 4. When choosing the collection set, special care is taken to assure that the amount of loaned memory required to + // hold the results of evacuation is smaller than the total memory occupied by the regions added to the collection + // set. We need to take these precautions because we do not know how much memory will be reclaimed by evacuation + // until after the collection set has been constructed. The algorithm is as follows: + // + // a. We feed into the algorithm (i) young available at the start of evacuation and (ii) the amount of memory + // loaned from old-gen that is available to hold the results of evacuation. + // b. As candidate regions are added into the young-gen collection set, we maintain accumulations of the amount + // of memory spanned by the collection set regions and the amount of memory that must be reserved to hold + // evacuation results (by multiplying live-data size by ShenandoahEvacWaste). We process candidate regions + // in order of decreasing amounts of garbage. We skip over (and do not include into the collection set) any + // regions that do not satisfy all of the following conditions: + // + // i. The amount of live data within the region as scaled by ShenandoahEvacWaste must fit within the + // relevant evacuation reserve (live data of old-gen regions must fit within the old-evac-reserve, live + // data of young-gen tenure-aged regions must fit within the advance promotion reserve, live data within + // other young-gen regions must fit within the youn-gen evacuation reserve). + // ii. The accumulation of memory consumed by evacuation must not exceed the accumulation of memory reclaimed + // through evacuation by more than young-gen available. + // iii. Other conditions may be enforced as appropriate for specific heuristics. + // + // Note that regions are considered for inclusion in the selection set in order of decreasing amounts of garbage. + // It is possible that a region with a larger amount of garbage will be rejected because it also has a larger + // amount of live data and some region that follows this region in candidate order is included in the collection + // set (because it has less live data and thus can fit within the evacuation limits even though it has less + // garbage). + + size_t young_evacuation_reserve = (young_generation->max_capacity() * ShenandoahEvacReserve) / 100; + // old evacuation can pack into existing partially used regions. young evacuation and loans for young allocations + // need to target regions that do not already hold any old-gen objects. Round down. + regions_available_to_loan = old_generation->free_unaffiliated_regions(); + consumed_by_advance_promotion = _heuristics->select_aged_regions(old_generation->available() - old_evacuation_reserve, + num_regions, preselected_regions); + size_t net_available_old_regions = + (old_generation->available() - old_evacuation_reserve - consumed_by_advance_promotion) / region_size_bytes; + + if (regions_available_to_loan > net_available_old_regions) { + regions_available_to_loan = net_available_old_regions; + } + + // Otherwise, regions_available_to_loan is less than net_available_old_regions because available memory is + // scattered between multiple partially used regions. + + if (young_evacuation_reserve > young_generation->available()) { + size_t short_fall = young_evacuation_reserve - young_generation->available(); + if (regions_available_to_loan * region_size_bytes >= short_fall) { + old_regions_loaned_for_young_evac = (short_fall + region_size_bytes - 1) / region_size_bytes; + regions_available_to_loan -= old_regions_loaned_for_young_evac; + } else { + old_regions_loaned_for_young_evac = regions_available_to_loan; + regions_available_to_loan = 0; + young_evacuation_reserve = young_generation->available() + old_regions_loaned_for_young_evac * region_size_bytes; + // In this case, there's no memory available for new allocations while evacuating and updating, unless we + // find more old-gen memory to borrow below. + } + } else { + old_regions_loaned_for_young_evac = 0; + } + // In generational mode, we may end up choosing a young collection set that contains so many promotable objects + // that there is not sufficient space in old generation to hold the promoted objects. That is ok because we have + // assured there is sufficient space in young generation to hold the rejected promotion candidates. These rejected + // promotion candidates will presumably be promoted in a future evacuation cycle. + heap->set_young_evac_reserve(young_evacuation_reserve); + collection_set->establish_preselected(preselected_regions); + } else { + // Not generational mode: limit young evac reserve by young available; no need to establish old_evac_reserve. + ShenandoahYoungGeneration* young_generation = heap->young_generation(); + size_t young_evac_reserve = (young_generation->soft_max_capacity() * ShenandoahEvacReserve) / 100; + if (young_evac_reserve > young_generation->available()) { + young_evac_reserve = young_generation->available(); + } + heap->set_young_evac_reserve(young_evac_reserve); + } +} + +// Having chosen the collection set, adjust the budgets for generatioal mode based on its composition. Note +// that young_generation->available() now knows about recently discovered immediate garbage. +void ShenandoahGeneration::adjust_evacuation_budgets(ShenandoahHeap* heap, ShenandoahCollectionSet* collection_set, + size_t old_regions_loaned_for_young_evac, size_t regions_available_to_loan, + size_t minimum_evacuation_reserve, size_t consumed_by_advance_promotion) { + if (heap->mode()->is_generational()) { + size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); + ShenandoahGeneration* old_generation = heap->old_generation(); + ShenandoahYoungGeneration* young_generation = heap->young_generation(); + size_t old_evacuated = collection_set->get_old_bytes_reserved_for_evacuation(); + size_t old_evacuated_committed = (size_t) (ShenandoahEvacWaste * old_evacuated); + size_t old_evacuation_reserve = heap->get_old_evac_reserve(); + // Immediate garbage found during choose_collection_set() is all young + size_t immediate_garbage = collection_set->get_immediate_trash(); + size_t old_available = old_generation->available(); + size_t young_available = young_generation->available() + immediate_garbage; + + assert(consumed_by_advance_promotion >= collection_set->get_young_bytes_to_be_promoted() * ShenandoahEvacWaste, + "Advance promotion should be at least young_bytes_to_be_promoted * ShenandoahEvacWaste"); + + assert(consumed_by_advance_promotion <= (collection_set->get_young_bytes_to_be_promoted() * ShenandoahEvacWaste * 33) / 32, + "Round-off errors should be less than 3.125%%, consumed by advance: " SIZE_FORMAT ", promoted: " SIZE_FORMAT, + consumed_by_advance_promotion, (size_t) (collection_set->get_young_bytes_to_be_promoted() * ShenandoahEvacWaste)); + + collection_set->abandon_preselected(); + if (old_evacuated_committed > old_evacuation_reserve) { + // This should only happen due to round-off errors when enforcing ShenandoahEvacWaste + assert(old_evacuated_committed <= (33 * old_evacuation_reserve) / 32, + "Round-off errors should be less than 3.125%%, committed: " SIZE_FORMAT ", reserved: " SIZE_FORMAT, + old_evacuated_committed, old_evacuation_reserve); + old_evacuated_committed = old_evacuation_reserve; + } else if (old_evacuated_committed < old_evacuation_reserve) { + // This may happen if the old-gen collection consumes less than full budget. + old_evacuation_reserve = old_evacuated_committed; + heap->set_old_evac_reserve(old_evacuation_reserve); + } + + // Recompute old_regions_loaned_for_young_evac because young-gen collection set may not need all the memory + // originally reserved. + size_t young_promoted = collection_set->get_young_bytes_to_be_promoted(); + size_t young_promoted_reserve_used = (size_t) (ShenandoahEvacWaste * young_promoted); + + size_t young_evacuated = collection_set->get_young_bytes_reserved_for_evacuation() - young_promoted; + size_t young_evacuated_reserve_used = (size_t) (ShenandoahEvacWaste * young_evacuated); + + heap->set_young_evac_reserve(young_evacuated_reserve_used); + + // Adjust old_regions_loaned_for_young_evac to feed into calculations of promoted_reserve + if (young_evacuated_reserve_used > young_available) { + size_t short_fall = young_evacuated_reserve_used - young_available; + + // region_size_bytes is a power of 2. loan an integral number of regions. + size_t revised_loan_for_young_evacuation = (short_fall + region_size_bytes - 1) / region_size_bytes; + + // Undo the previous loan + regions_available_to_loan += old_regions_loaned_for_young_evac; + old_regions_loaned_for_young_evac = revised_loan_for_young_evacuation; + // And make a new loan + assert(regions_available_to_loan > old_regions_loaned_for_young_evac, "Cannot loan regions that we do not have"); + regions_available_to_loan -= old_regions_loaned_for_young_evac; + } else { + // Undo the prevous loan + regions_available_to_loan += old_regions_loaned_for_young_evac; + old_regions_loaned_for_young_evac = 0; + } + + size_t old_bytes_loaned_for_young_evac = old_regions_loaned_for_young_evac * region_size_bytes; + size_t old_bytes_reserved_for_alloc_supplement = 0; + size_t old_regions_reserved_for_alloc_supplement = 0; + // Need to enforce that old_evacuated_committed + old_bytes_loaned_for_young_evac >= minimum_evacuation_reserve + // in order to prevent promotion reserve from violating minimum evacuation reserve. + if (old_evacuated_committed + old_bytes_loaned_for_young_evac < minimum_evacuation_reserve) { + // Reserve some of the regions available to loan for use as allocation supplement to assure memory not consumed by promotion + size_t excess_bytes = minimum_evacuation_reserve - (old_evacuated_committed + old_bytes_loaned_for_young_evac); + size_t excess_regions = (excess_bytes - 1 + region_size_bytes) / region_size_bytes; + if (regions_available_to_loan <= excess_regions) { + excess_regions = regions_available_to_loan; + // Since we can't reserve entire excess for alloc supplement, pretend more is consumed by old-evacuation + old_evacuated_committed = + minimum_evacuation_reserve - old_bytes_loaned_for_young_evac - excess_regions * region_size_bytes; + } + regions_available_to_loan -= excess_regions; + old_bytes_reserved_for_alloc_supplement = excess_regions * region_size_bytes; + old_regions_reserved_for_alloc_supplement = excess_regions; + } + + // Limit promoted_reserve so that we can set aside memory to be loaned from old-gen to young-gen. This + // value is not "critical". If we underestimate, certain promotions will simply be deferred. If we put + // "all the rest" of old-gen memory into the promotion reserve, we'll have nothing left to loan to young-gen + // during the evac and update phases of GC. So we "limit" the sizes of the promotion budget to be the smaller of: + // + // 1. old_gen->available - (old_evacuation_committed + old_bytes_loaned_for_young_evac + consumed_by_advance_promotion) + // 2. young bytes reserved for evacuation + + assert(old_available > old_evacuated_committed, "Cannot evacuate more than available"); + assert(old_available > old_evacuated_committed + old_bytes_loaned_for_young_evac, + "Cannot loan young evac more than available"); + assert(old_available > old_evacuated_committed + old_bytes_loaned_for_young_evac + consumed_by_advance_promotion, + "Cannot promote more than available"); + assert(old_available > (old_evacuated_committed + old_bytes_loaned_for_young_evac + + consumed_by_advance_promotion + old_bytes_reserved_for_alloc_supplement), + "Cannot loan for alloc supplement more than available"); + + size_t promotion_reserve = old_available - (old_evacuated_committed + consumed_by_advance_promotion + + old_bytes_loaned_for_young_evac + old_bytes_reserved_for_alloc_supplement); + + // We experimented with constraining promoted_reserve to be no larger than 4 times the size of previously_promoted, + // but this constraint was too limiting, resulting in failure of legitimate promotions. This was tried before we + // had special handling in place for advance promotion. We should retry now that advance promotion is handled + // specially. + + // We had also experimented with constraining promoted_reserve to be no more than young_evacuation_committed + // divided by promotion_divisor, where: + // size_t promotion_divisor = (0x02 << InitialTenuringThreshold) - 1; + // This also was found to be too limiting, resulting in failure of legitimate promotions. + // + // Both experiments were conducted in the presence of other bugs which could have been the root cause for + // the failures identified above as being "too limiting". TODO: conduct new experiments with the more limiting + // values of young_evacuation_reserved_used. + + // young_evacuation_reserve_used already excludes bytes known to be promoted, which equals consumed_by_advance_promotion + if (young_evacuated_reserve_used < promotion_reserve) { + // Shrink promotion_reserve if it is larger than the memory to be consumed by evacuating all young objects in + // collection set, including anticipated waste. There's no benefit in using a larger promotion_reserve. + // young_evacuation_reserve_used does not include live memory within tenure-aged regions. + promotion_reserve = young_evacuated_reserve_used; + } + + assert(old_available >= (promotion_reserve + old_evacuated_committed + old_bytes_loaned_for_young_evac + + consumed_by_advance_promotion + old_bytes_reserved_for_alloc_supplement), + "Budget exceeds available old-gen memory"); + log_debug(gc)("Old available: " SIZE_FORMAT ", Original promotion reserve: " SIZE_FORMAT ", Old evacuation reserve: " + SIZE_FORMAT ", Advance promotion reserve supplement: " SIZE_FORMAT + ", Old loaned for young evacuation: " SIZE_FORMAT ", Old reserved for alloc supplement: " SIZE_FORMAT, + old_available, promotion_reserve, old_evacuated_committed, consumed_by_advance_promotion, + old_regions_loaned_for_young_evac * region_size_bytes, old_bytes_reserved_for_alloc_supplement); + + promotion_reserve += consumed_by_advance_promotion; + heap->set_promoted_reserve(promotion_reserve); + heap->reset_promoted_expended(); + if (collection_set->get_old_bytes_reserved_for_evacuation() == 0) { + // Setting old evacuation reserve to zero denotes that there is no old-gen evacuation in this pass. + heap->set_old_evac_reserve(0); + } + + size_t old_gen_usage_base = old_generation->used() - collection_set->get_old_garbage(); + heap->capture_old_usage(old_gen_usage_base); + + // Compute the evacuation supplement, which is extra memory borrowed from old-gen that can be allocated + // by mutators while GC is working on evacuation and update-refs. This memory can be temporarily borrowed + // from old-gen allotment, then repaid at the end of update-refs from the recycled collection set. After + // we have computed the collection set based on the parameters established above, we can make additional + // loans based on our knowledge of the collection set to determine how much allocation we can allow + // during the evacuation and update-refs phases of execution. The total available supplement is the smaller of: + // + // 1. old_gen->available() - + // (promotion_reserve + old_evacuation_commitment + old_bytes_loaned_for_young_evac) + // 2. The replenishment budget (number of regions in collection set - the number of regions already + // under lien for the young_evacuation_reserve) + // + + // Regardless of how many regions may be available to be loaned, we can loan no more regions than + // the total number of young regions to be evacuated. Call this the regions_for_runway. + + size_t young_regions_evacuated = collection_set->get_young_region_count(); + size_t regions_for_runway = 0; + size_t already_loaned_regions = old_regions_loaned_for_young_evac + old_regions_reserved_for_alloc_supplement; + if (already_loaned_regions == 0) { + regions_for_runway = young_regions_evacuated; + } else if (young_regions_evacuated > already_loaned_regions) { + regions_for_runway = young_regions_evacuated - already_loaned_regions; + } else { + regions_for_runway = 0; + } + + if (regions_available_to_loan > regions_for_runway) { + regions_available_to_loan -= regions_for_runway; + } else { + regions_for_runway = regions_available_to_loan; + regions_available_to_loan = 0; + } + + size_t allocation_supplement = regions_for_runway * region_size_bytes + old_bytes_reserved_for_alloc_supplement; + heap->set_alloc_supplement_reserve(allocation_supplement); + + // TODO: young_available, which feeds into alloc_budget_evac_and_update is lacking memory available within + // existing young-gen regions that were not selected for the collection set. Add this in and adjust the + // log message (where it says "empty-region allocation budget"). + + log_debug(gc)("Memory reserved for young evacuation: " SIZE_FORMAT "%s for evacuating " SIZE_FORMAT + "%s out of young available: " SIZE_FORMAT "%s", + byte_size_in_proper_unit(young_evacuated_reserve_used), + proper_unit_for_byte_size(young_evacuated_reserve_used), + byte_size_in_proper_unit(young_evacuated), proper_unit_for_byte_size(young_evacuated), + byte_size_in_proper_unit(young_available), proper_unit_for_byte_size(young_available)); + + log_debug(gc)("Memory reserved for old evacuation: " SIZE_FORMAT "%s for evacuating " SIZE_FORMAT + "%s out of old available: " SIZE_FORMAT "%s", + byte_size_in_proper_unit(old_evacuated), proper_unit_for_byte_size(old_evacuated), + byte_size_in_proper_unit(old_evacuated), proper_unit_for_byte_size(old_evacuated), + byte_size_in_proper_unit(old_available), proper_unit_for_byte_size(old_available)); + + assert(old_available > old_evacuation_reserve + promotion_reserve + old_bytes_loaned_for_young_evac + allocation_supplement, + "old_available must be larger than accumulated reserves"); + + size_t regular_promotion = promotion_reserve - consumed_by_advance_promotion; + size_t excess = + old_available - (old_evacuation_reserve + promotion_reserve + old_bytes_loaned_for_young_evac + allocation_supplement); + log_info(gc, ergo)("Old available: " SIZE_FORMAT "%s is partitioned into old evacuation budget: " SIZE_FORMAT + "%s, aged region promotion budget: " SIZE_FORMAT + "%s, regular region promotion budget: " SIZE_FORMAT + "%s, loaned for young evacuation: " SIZE_FORMAT + "%s, loaned for young allocations: " SIZE_FORMAT + "%s, excess: " SIZE_FORMAT "%s", + byte_size_in_proper_unit(old_available), proper_unit_for_byte_size(old_available), + byte_size_in_proper_unit(old_evacuation_reserve), proper_unit_for_byte_size(old_evacuation_reserve), + byte_size_in_proper_unit(consumed_by_advance_promotion), + proper_unit_for_byte_size(consumed_by_advance_promotion), + byte_size_in_proper_unit(regular_promotion), proper_unit_for_byte_size(regular_promotion), + byte_size_in_proper_unit(old_bytes_loaned_for_young_evac), + proper_unit_for_byte_size(old_bytes_loaned_for_young_evac), + byte_size_in_proper_unit(allocation_supplement), proper_unit_for_byte_size(allocation_supplement), + byte_size_in_proper_unit(excess), proper_unit_for_byte_size(excess)); + } + // else, not generational: no evacuation budget adjustments required +} + +void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) { ShenandoahHeap* heap = ShenandoahHeap::heap(); ShenandoahCollectionSet* collection_set = heap->collection_set(); size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); @@ -244,199 +663,20 @@ void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) } { - ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::choose_cset : - ShenandoahPhaseTimings::degen_gc_choose_cset); - ShenandoahHeapLocker locker(heap->lock()); - heap->collection_set()->clear(); - - size_t minimum_evacuation_reserve = ShenandoahOldCompactionReserve * region_size_bytes; - size_t avail_evac_reserve_for_loan_to_young_gen = 0; - size_t old_regions_loaned_for_young_evac = 0; - size_t regions_available_to_loan = 0; - size_t old_evacuation_reserve = 0; - size_t num_regions = heap->num_regions(); - size_t consumed_by_advance_promotion = 0; - bool* preselected_regions = NEW_C_HEAP_ARRAY(bool, num_regions, mtGC); - for (unsigned int i = 0; i < num_regions; i++) { - preselected_regions[i] = false; - } - + size_t old_regions_loaned_for_young_evac, regions_available_to_loan, minimum_evacuation_reserve, consumed_by_advance_promotion; + bool* preselected_regions = nullptr; if (heap->mode()->is_generational()) { - ShenandoahGeneration* old_generation = heap->old_generation(); - ShenandoahYoungGeneration* young_generation = heap->young_generation(); - - // During initialization and phase changes, it is more likely that fewer objects die young and old-gen - // memory is not yet full (or is in the process of being replaced). During these times especially, it - // is beneficial to loan memory from old-gen to young-gen during the evacuation and update-refs phases - // of execution. - - // Calculate EvacuationReserve before PromotionReserve. Evacuation is more critical than promotion. - // If we cannot evacuate old-gen, we will not be able to reclaim old-gen memory. Promotions are less - // critical. If we cannot promote, there may be degradation of young-gen memory because old objects - // accumulate there until they can be promoted. This increases the young-gen marking and evacuation work. - - // Do not fill up old-gen memory with promotions. Reserve some amount of memory for compaction purposes. - ShenandoahOldHeuristics* old_heuristics = heap->old_heuristics(); - if (old_heuristics->unprocessed_old_collection_candidates() > 0) { - - // Compute old_evacuation_reserve: how much memory are we reserving to hold the results of - // evacuating old-gen heap regions? In order to sustain a consistent pace of young-gen collections, - // the goal is to maintain a consistent value for this parameter (when the candidate set is not - // empty). This value is the minimum of: - // 1. old_gen->available() - // 2. old-gen->capacity() * ShenandoahOldEvacReserve) / 100 - // (e.g. old evacuation should be no larger than 5% of old_gen capacity) - // 3. ((young_gen->capacity * ShenandoahEvacReserve / 100) * ShenandoahOldEvacRatioPercent) / 100 - // (e.g. old evacuation should be no larger than 12% of young-gen evacuation) - - old_evacuation_reserve = old_generation->available(); - assert(old_evacuation_reserve > minimum_evacuation_reserve, "Old-gen available has not been preserved!"); - size_t old_evac_reserve_max = old_generation->soft_max_capacity() * ShenandoahOldEvacReserve / 100; - if (old_evac_reserve_max < old_evacuation_reserve) { - old_evacuation_reserve = old_evac_reserve_max; - } - size_t young_evac_reserve_max = - (((young_generation->soft_max_capacity() * ShenandoahEvacReserve) / 100) * ShenandoahOldEvacRatioPercent) / 100; - if (young_evac_reserve_max < old_evacuation_reserve) { - old_evacuation_reserve = young_evac_reserve_max; - } + preselected_regions = (bool*) alloca(heap->num_regions() * sizeof(bool)); + for (unsigned int i = 0; i < heap->num_regions(); i++) { + preselected_regions[i] = false; } - - if (minimum_evacuation_reserve > old_generation->available()) { - // Due to round-off errors during enforcement of minimum_evacuation_reserve during previous GC passes, - // there can be slight discrepancies here. - minimum_evacuation_reserve = old_generation->available(); - } - if (old_evacuation_reserve < minimum_evacuation_reserve) { - // Even if there's nothing to be evacuated on this cycle, we still need to reserve this memory for future - // evacuations. It is ok to loan this memory to young-gen if we don't need it for evacuation on this pass. - avail_evac_reserve_for_loan_to_young_gen = minimum_evacuation_reserve - old_evacuation_reserve; - old_evacuation_reserve = minimum_evacuation_reserve; - } - - heap->set_old_evac_reserve(old_evacuation_reserve); - heap->reset_old_evac_expended(); - - // Compute the young evauation reserve: This is how much memory is available for evacuating young-gen objects. - // We ignore the possible effect of promotions, which reduce demand for young-gen evacuation memory. - // - // TODO: We could give special treatment to the regions that have reached promotion age, because we know their - // live data is entirely eligible for promotion. This knowledge can feed both into calculations of young-gen - // evacuation reserve and promotion reserve. - // - // young_evacuation_reserve for young generation: how much memory are we reserving to hold the results - // of evacuating young collection set regions? This is typically smaller than the total amount - // of available memory, and is also smaller than the total amount of marked live memory within - // young-gen. This value is the smaller of - // - // 1. (young_gen->capacity() * ShenandoahEvacReserve) / 100 - // 2. (young_gen->available() + old_gen_memory_available_to_be_loaned - // - // ShenandoahEvacReserve represents the configured taget size of the evacuation region. We can only honor - // this target if there is memory available to hold the evacuations. Memory is available if it is already - // free within young gen, or if it can be borrowed from old gen. Since we have not yet chosen the collection - // sets, we do not yet know the exact accounting of how many regions will be freed by this collection pass. - // What we do know is that there will be at least one evacuated young-gen region for each old-gen region that - // is loaned to the evacuation effort (because regions to be collected consume more memory than the compacted - // regions that will replace them). In summary, if there are old-gen regions that are available to hold the - // results of young-gen evacuations, it is safe to loan them for this purpose. At this point, we have not yet - // established a promoted_reserve. We'll do that after we choose the collection set and analyze its impact - // on available memory. - // - // We do not know the evacuation_supplement until after we have computed the collection set. It is not always - // the case that young-regions inserted into the collection set will result in net decrease of in-use regions - // because ShenandoahEvacWaste times multiplied by memory within the region may be larger than the region size. - // The problem is especially relevant to regions that have been inserted into the collection set because they have - // reached tenure age. These regions tend to have much higher utilization (e.g. 95%). These regions also offer - // a unique opportunity because we know that every live object contained within the region is elgible to be - // promoted. Thus, the following implementation treats these regions specially: - // - // 1. Before beginning collection set selection, we tally the total amount of live memory held within regions - // that are known to have reached tenure age. If this memory times ShenandoahEvacWaste is available within - // old-gen memory, establish an advance promotion reserve to hold all or some percentage of these objects. - // This advance promotion reserve is excluded from memory available for holding old-gen evacuations and cannot - // be "loaned" to young gen. - // - // 2. Tenure-aged regions are included in the collection set iff their evacuation size * ShenandoahEvacWaste fits - // within the advance promotion reserve. It is counter productive to evacuate these regions if they cannot be - // evacuated directly into old-gen memory. So if there is not sufficient memory to hold copies of their - // live data right now, we'll just let these regions remain in young for now, to be evacuated by a subsequent - // evacuation pass. - // - // 3. Next, we calculate a young-gen evacuation budget, which is the smaller of the two quantities mentioned - // above. old_gen_memory_available_to_be_loaned is calculated as: - // old_gen->available - (advance-promotion-reserve + old-gen_evacuation_reserve) - // - // 4. When choosing the collection set, special care is taken to assure that the amount of loaned memory required to - // hold the results of evacuation is smaller than the total memory occupied by the regions added to the collection - // set. We need to take these precautions because we do not know how much memory will be reclaimed by evacuation - // until after the collection set has been constructed. The algorithm is as follows: - // - // a. We feed into the algorithm (i) young available at the start of evacuation and (ii) the amount of memory - // loaned from old-gen that is available to hold the results of evacuation. - // b. As candidate regions are added into the young-gen collection set, we maintain accumulations of the amount - // of memory spanned by the collection set regions and the amount of memory that must be reserved to hold - // evacuation results (by multiplying live-data size by ShenandoahEvacWaste). We process candidate regions - // in order of decreasing amounts of garbage. We skip over (and do not include into the collection set) any - // regions that do not satisfy all of the following conditions: - // - // i. The amount of live data within the region as scaled by ShenandoahEvacWaste must fit within the - // relevant evacuation reserve (live data of old-gen regions must fit within the old-evac-reserve, live - // data of young-gen tenure-aged regions must fit within the advance promotion reserve, live data within - // other young-gen regions must fit within the youn-gen evacuation reserve). - // ii. The accumulation of memory consumed by evacuation must not exceed the accumulation of memory reclaimed - // through evacuation by more than young-gen available. - // iii. Other conditions may be enforced as appropriate for specific heuristics. - // - // Note that regions are considered for inclusion in the selection set in order of decreasing amounts of garbage. - // It is possible that a region with a larger amount of garbage will be rejected because it also has a larger - // amount of live data and some region that follows this region in candidate order is included in the collection - // set (because it has less live data and thus can fit within the evacuation limits even though it has less - // garbage). - - size_t young_evacuation_reserve = (young_generation->max_capacity() * ShenandoahEvacReserve) / 100; - // old evacuation can pack into existing partially used regions. young evacuation and loans for young allocations - // need to target regions that do not already hold any old-gen objects. Round down. - regions_available_to_loan = old_generation->free_unaffiliated_regions(); - consumed_by_advance_promotion = _heuristics->select_aged_regions(old_generation->available() - old_evacuation_reserve, - num_regions, preselected_regions); - size_t net_available_old_regions = - (old_generation->available() - old_evacuation_reserve - consumed_by_advance_promotion) / region_size_bytes; - - if (regions_available_to_loan > net_available_old_regions) { - regions_available_to_loan = net_available_old_regions; - } - // Otherwise, regions_available_to_loan is less than net_available_old_regions because available memory is - // scattered between multiple partially used regions. - - if (young_evacuation_reserve > young_generation->available()) { - size_t short_fall = young_evacuation_reserve - young_generation->available(); - if (regions_available_to_loan * region_size_bytes >= short_fall) { - old_regions_loaned_for_young_evac = (short_fall + region_size_bytes - 1) / region_size_bytes; - regions_available_to_loan -= old_regions_loaned_for_young_evac; - } else { - old_regions_loaned_for_young_evac = regions_available_to_loan; - regions_available_to_loan = 0; - young_evacuation_reserve = young_generation->available() + old_regions_loaned_for_young_evac * region_size_bytes; - } - } else { - old_regions_loaned_for_young_evac = 0; - } - // In generational mode, we may end up choosing a young collection set that contains so many promotable objects - // that there is not sufficient space in old generation to hold the promoted objects. That is ok because we have - // assured there is sufficient space in young generation to hold the rejected promotion candidates. These rejected - // promotion candidates will presumably be promoted in a future evacuation cycle. - heap->set_young_evac_reserve(young_evacuation_reserve); - } else { - // Not generational mode: limit young evac reserve by young available; no need to establish old_evac_reserve. - ShenandoahYoungGeneration* young_generation = heap->young_generation(); - size_t young_evac_reserve = (young_generation->soft_max_capacity() * ShenandoahEvacReserve) / 100; - if (young_evac_reserve > young_generation->available()) { - young_evac_reserve = young_generation->available(); - } - heap->set_young_evac_reserve(young_evac_reserve); } + ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::choose_cset : + ShenandoahPhaseTimings::degen_gc_choose_cset); + + ShenandoahHeapLocker locker(heap->lock()); + collection_set->clear(); // TODO: young_available can include available (between top() and end()) within each young region that is not // part of the collection set. Making this memory available to the young_evacuation_reserve allows a larger // young collection set to be chosen when available memory is under extreme pressure. Implementing this "improvement" @@ -445,158 +685,15 @@ void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) // non-empty regions that are not selected as part of the collection set can be allocated by the mutator while // GC is evacuating and updating references. - collection_set->establish_preselected(preselected_regions); + // Budgeting parameters to compute_evacuation_budgets are passed by reference. + compute_evacuation_budgets(heap, preselected_regions, collection_set, old_regions_loaned_for_young_evac, + regions_available_to_loan, minimum_evacuation_reserve, consumed_by_advance_promotion); _heuristics->choose_collection_set(collection_set, heap->old_heuristics()); - collection_set->abandon_preselected(); - FREE_C_HEAP_ARRAY(bool, preselected_regions); - - // At this point, young_generation->available() knows about recently discovered immediate garbage. We also - // know the composition of the chosen collection set. - - if (heap->mode()->is_generational()) { - ShenandoahGeneration* old_generation = heap->old_generation(); - ShenandoahYoungGeneration* young_generation = heap->young_generation(); - size_t old_evacuation_committed = (size_t) (ShenandoahEvacWaste * - collection_set->get_old_bytes_reserved_for_evacuation()); - size_t immediate_garbage_regions = collection_set->get_immediate_trash() / region_size_bytes; - - if (old_evacuation_committed > old_evacuation_reserve) { - // This should only happen due to round-off errors when enforcing ShenandoahEvacWaste - assert(old_evacuation_committed < (33 * old_evacuation_reserve) / 32, "Round-off errors should be less than 3.125%%"); - old_evacuation_committed = old_evacuation_reserve; - } - - // Recompute old_regions_loaned_for_young_evac because young-gen collection set may not need all the memory - // originally reserved. - - size_t young_evacuation_reserve_used = - collection_set->get_young_bytes_reserved_for_evacuation() - collection_set->get_young_bytes_to_be_promoted(); - young_evacuation_reserve_used = (size_t) (ShenandoahEvacWaste * young_evacuation_reserve_used); - heap->set_young_evac_reserve(young_evacuation_reserve_used); - - // Adjust old_regions_loaned_for_young_evac to feed into calculations of promoted_reserve - if (young_evacuation_reserve_used > young_generation->available()) { - size_t short_fall = young_evacuation_reserve_used - young_generation->available(); - - // region_size_bytes is a power of 2. loan an integral number of regions. - size_t revised_loan_for_young_evacuation = (short_fall + region_size_bytes - 1) / region_size_bytes; - - // Undo the previous loan - regions_available_to_loan += old_regions_loaned_for_young_evac; - old_regions_loaned_for_young_evac = revised_loan_for_young_evacuation; - // And make a new loan - assert(regions_available_to_loan > old_regions_loaned_for_young_evac, "Cannot loan regions that we do not have"); - regions_available_to_loan -= old_regions_loaned_for_young_evac; - } else { - // Undo the prevous loan - regions_available_to_loan += old_regions_loaned_for_young_evac; - old_regions_loaned_for_young_evac = 0; - } - - size_t old_bytes_loaned = old_regions_loaned_for_young_evac * region_size_bytes; - // Need to enforce that old_evacuation_committed + old_bytes_loaned >= minimum_evacuation_reserve - // in order to prevent promotion reserve from violating minimum evacuation reserve. - if (old_evacuation_committed + old_bytes_loaned < minimum_evacuation_reserve) { - // Pretend the old_evacuation_commitment is larger than what will be evacuated to assure that promotions - // do not fill the minimum_evacuation_reserve. Note that regions loaned from old-gen will be returned - // to old-gen before we start a subsequent evacuation. - old_evacuation_committed = minimum_evacuation_reserve - old_bytes_loaned; - } - - // Limit promoted_reserve so that we can set aside memory to be loaned from old-gen to young-gen. This - // value is not "critical". If we underestimate, certain promotions will simply be deferred. If we put - // "all the rest" of old-gen memory into the promotion reserve, we'll have nothing left to loan to young-gen - // during the evac and update phases of GC. So we "limit" the sizes of the promotion budget to be the smaller of: - // - // 1. old_gen->available - (old_evacuation_committed + old_bytes_loaned + consumed_by_advance_promotion) - // 2. young bytes reserved for evacuation - - assert(old_generation->available() > old_evacuation_committed, "Cannot evacuate more than available"); - assert(old_generation->available() > old_evacuation_committed + old_bytes_loaned, "Cannot loan more than available"); - assert(old_generation->available() > old_evacuation_committed + old_bytes_loaned + consumed_by_advance_promotion, - "Cannot promote more than available"); - - size_t old_avail = old_generation->available(); - size_t promotion_reserve = old_avail - (old_evacuation_committed + consumed_by_advance_promotion + old_bytes_loaned); - - // We experimented with constraining promoted_reserve to be no larger than 4 times the size of previously_promoted, - // but this constraint was too limiting, resulting in failure of legitimate promotions. - - // We had also experimented with constraining promoted_reserve to be no more than young_evacuation_committed - // divided by promotion_divisor, where: - // size_t promotion_divisor = (0x02 << InitialTenuringThreshold) - 1; - // This also was found to be too limiting, resulting in failure of legitimate promotions. - // - // Both experiments were conducted in the presence of other bugs which could have been the root cause for - // the failures identified above as being "too limiting". TODO: conduct new experiments with the more limiting - // values of young_evacuation_reserved_used. - young_evacuation_reserve_used -= consumed_by_advance_promotion; - if (young_evacuation_reserve_used < promotion_reserve) { - // Shrink promotion_reserve if its larger than the memory to be consumed by evacuating all young objects in - // collection set, including anticipated waste. There's no benefit in using a larger promotion_reserve. - promotion_reserve = young_evacuation_reserve_used; - } - - assert(old_avail >= promotion_reserve + old_evacuation_committed + old_bytes_loaned + consumed_by_advance_promotion, - "Budget exceeds available old-gen memory"); - log_info(gc, ergo)("Old available: " SIZE_FORMAT ", Original promotion reserve: " SIZE_FORMAT ", Old evacuation reserve: " - SIZE_FORMAT ", Advance promotion reserve supplement: " SIZE_FORMAT ", Old loaned to young: " SIZE_FORMAT, - old_avail, promotion_reserve, old_evacuation_committed, consumed_by_advance_promotion, - old_regions_loaned_for_young_evac * region_size_bytes); - promotion_reserve += consumed_by_advance_promotion; - heap->set_promoted_reserve(promotion_reserve); - heap->reset_promoted_expended(); - if (collection_set->get_old_bytes_reserved_for_evacuation() == 0) { - // Setting old evacuation reserve to zero denotes that there is no old-gen evacuation in this pass. - heap->set_old_evac_reserve(0); - } - - size_t old_gen_usage_base = old_generation->used() - collection_set->get_old_garbage(); - heap->capture_old_usage(old_gen_usage_base); - - // Compute the evacuation supplement, which is extra memory borrowed from old-gen that can be allocated - // by mutators while GC is working on evacuation and update-refs. This memory can be temporarily borrowed - // from old-gen allotment, then repaid at the end of update-refs from the recycled collection set. After - // we have computed the collection set based on the parameters established above, we can make additional - // loans based on our knowledge of the collection set to determine how much allocation we can allow - // during the evacuation and update-refs phases of execution. The total available supplement is the smaller of: - // - // 1. old_gen->available() - - // (promotion_reserve + old_evacuation_commitment + old_bytes_loaned) - // 2. The replenishment budget (number of regions in collection set - the number of regions already - // under lien for the young_evacuation_reserve) - // - - size_t young_regions_evacuated = collection_set->get_young_region_count(); - size_t regions_for_runway = 0; - if (young_regions_evacuated > old_regions_loaned_for_young_evac) { - regions_for_runway = young_regions_evacuated - old_regions_loaned_for_young_evac; - old_regions_loaned_for_young_evac = young_regions_evacuated; - regions_available_to_loan -= regions_for_runway; - } - - size_t allocation_supplement = regions_for_runway * region_size_bytes; - heap->set_alloc_supplement_reserve(allocation_supplement); - - size_t promotion_budget = heap->get_promoted_reserve(); - size_t old_evac_budget = heap->get_old_evac_reserve(); - size_t alloc_budget_evac_and_update = allocation_supplement + young_generation->available(); - - // TODO: young_available, which feeds into alloc_budget_evac_and_update is lacking memory available within - // existing young-gen regions that were not selected for the collection set. Add this in and adjust the - // log message (where it says "empty-region allocation budget"). - - log_info(gc, ergo)("Memory reserved for evacuation and update-refs includes promotion budget: " SIZE_FORMAT - "%s, young evacuation budget: " SIZE_FORMAT "%s, old evacuation budget: " SIZE_FORMAT - "%s, empty-region allocation budget: " SIZE_FORMAT "%s, including supplement: " SIZE_FORMAT "%s", - byte_size_in_proper_unit(promotion_budget), proper_unit_for_byte_size(promotion_budget), - byte_size_in_proper_unit(young_evacuation_reserve_used), - proper_unit_for_byte_size(young_evacuation_reserve_used), - byte_size_in_proper_unit(old_evac_budget), proper_unit_for_byte_size(old_evac_budget), - byte_size_in_proper_unit(alloc_budget_evac_and_update), - proper_unit_for_byte_size(alloc_budget_evac_and_update), - byte_size_in_proper_unit(allocation_supplement), proper_unit_for_byte_size(allocation_supplement)); + if (!collection_set->is_empty()) { + adjust_evacuation_budgets(heap, collection_set, old_regions_loaned_for_young_evac, regions_available_to_loan, + minimum_evacuation_reserve, consumed_by_advance_promotion); } + // otherwise, this is an abbreviated cycle and we make no use of evacuation budgets. } { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index ebeda4f9cb4d8..bc0907152ee39 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -56,6 +56,18 @@ class ShenandoahGeneration : public CHeapObj { size_t _adjusted_capacity; ShenandoahHeuristics* _heuristics; + +private: + // Compute evacuation budgets prior to choosing collection set. + void compute_evacuation_budgets(ShenandoahHeap* heap, bool* preselected_regions, ShenandoahCollectionSet* collection_set, + size_t &old_regions_loaned_for_young_evac, size_t ®ions_available_to_loan, + size_t &minimum_evacuation_reserve, size_t &consumed_by_advance_promotion); + + // Adjust evacuation budgets after choosing collection set. + void adjust_evacuation_budgets(ShenandoahHeap* heap, ShenandoahCollectionSet* collection_set, + size_t old_regions_loaned_for_young_evac, size_t regions_available_to_loan, + size_t minimum_evacuation_reserve, size_t consumed_by_advance_promotion); + public: ShenandoahGeneration(GenerationMode generation_mode, uint max_workers, size_t max_capacity, size_t soft_max_capacity); ~ShenandoahGeneration(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 13f33a73c360b..b5473037131ea 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -657,17 +657,20 @@ bool ShenandoahHeap::is_gc_generation_young() const { // If this bound is smaller than NewSize, it supersedes, // resulting in a fixed size given by MaxNewSize. size_t ShenandoahHeap::young_generation_capacity(size_t capacity) { - if (FLAG_IS_CMDLINE(NewSize) && !FLAG_IS_CMDLINE(MaxNewSize) && !FLAG_IS_CMDLINE(NewRatio)) { - capacity = MIN2(NewSize, capacity); - } else { - capacity /= NewRatio + 1; - if (FLAG_IS_CMDLINE(NewSize)) { - capacity = MAX2(NewSize, capacity); - } - if (FLAG_IS_CMDLINE(MaxNewSize)) { - capacity = MIN2(MaxNewSize, capacity); + if (strcmp(ShenandoahGCMode, "generational") == 0) { + if (FLAG_IS_CMDLINE(NewSize) && !FLAG_IS_CMDLINE(MaxNewSize) && !FLAG_IS_CMDLINE(NewRatio)) { + capacity = MIN2(NewSize, capacity); + } else { + capacity /= NewRatio + 1; + if (FLAG_IS_CMDLINE(NewSize)) { + capacity = MAX2(NewSize, capacity); + } + if (FLAG_IS_CMDLINE(MaxNewSize)) { + capacity = MIN2(MaxNewSize, capacity); + } } } + // else, make no adjustment to global capacity return capacity; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp index d40d8d709d27c..51219532c09af 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp @@ -686,6 +686,7 @@ size_t ShenandoahHeap::capture_old_usage(size_t old_usage) { } void ShenandoahHeap::set_previous_promotion(size_t promoted_bytes) { + shenandoah_assert_heaplocked(); _previous_promotion = promoted_bytes; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp b/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp index 18b1ead0343e5..b5a107857a3ba 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp @@ -537,7 +537,6 @@ void ShenandoahReferenceProcessor::enqueue_references(bool concurrent) { // Nothing to enqueue return; } - if (!concurrent) { // When called from mark-compact or degen-GC, the locking is done by the VMOperation, enqueue_references_locked(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp index e395106bce03b..3b0101188dc2a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp @@ -101,6 +101,15 @@ "be taken for collection.") \ range(0,100) \ \ + product(uintx, ShenandoahIgnoreGarbageThreshold, 5, EXPERIMENTAL, \ + "When less than this amount of garbage (as a percentage of " \ + "region size) exists within a region, the region will not be " \ + "added to the collection set, even when the heuristic has " \ + "chosen to aggressively add regions with less than " \ + "ShenandoahGarbageThreshold amount of garbage into the " \ + "collection set.") \ + range(0,100) \ + \ product(uintx, ShenandoahInitFreeThreshold, 70, EXPERIMENTAL, \ "When less than this amount of memory is free within the" \ "heap or generation, trigger a learning cycle if we are " \ @@ -499,22 +508,6 @@ "Fix references with load reference barrier. Disabling this " \ "might degrade performance.") \ \ - product(uintx, ShenandoahTenuredRegionUsageBias, 16, EXPERIMENTAL, \ - "The collection set is comprised of heap regions that contain " \ - "the greatest amount of garbage. " \ - "For purposes of selecting regions to be included in the " \ - "collection set, regions that have reached the tenure age will " \ - "be treated as if their contained garbage is the contained " \ - "garbage multiplied by ShenandoahTenuredRegionUsageBias as " \ - "many times as the age of the region meets or exceeds " \ - "tenure age. For example, if tenure age is 7, " \ - "the region age is 9, ShenandoahTenuredRegionUsageBias is " \ - "16, and the region is 12.5% garbage, this region " \ - "will by treated as if its garbage content is " \ - "12.5% * 16 * 16 * 16 = 51,200% when comparing this region " \ - " to untenured regions.") \ - range(1,128) \ - \ product(uintx, ShenandoahBorrowPercent, 30, EXPERIMENTAL, \ "During evacuation and reference updating in generational " \ "mode, new allocations are allowed to borrow from old-gen " \ From 16a9ab63be2591fd4133a204a000cb3406afdede Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Mon, 1 Aug 2022 22:15:30 +0000 Subject: [PATCH 142/254] Load balance remembered set scanning Reviewed-by: wkemper --- .../gc/shenandoah/shenandoahConcurrentGC.cpp | 2 +- .../gc/shenandoah/shenandoahGeneration.cpp | 6 +- .../gc/shenandoah/shenandoahGeneration.hpp | 2 +- .../share/gc/shenandoah/shenandoahHeap.cpp | 192 ++++++++++++----- .../share/gc/shenandoah/shenandoahSTWMark.cpp | 2 +- .../shenandoah/shenandoahScanRemembered.cpp | 193 +++++++++++++++++- .../shenandoah/shenandoahScanRemembered.hpp | 109 +++++++++- .../shenandoahScanRemembered.inline.hpp | 180 +++++++++++++--- 8 files changed, 580 insertions(+), 106 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index 54b168937f96e..88aa3e70726a0 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -122,7 +122,7 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) { // Concurrent remembered set scanning if (_generation->generation_mode() == YOUNG) { ShenandoahConcurrentPhase gc_phase("Concurrent remembered set scanning", ShenandoahPhaseTimings::init_scan_rset); - _generation->scan_remembered_set(); + _generation->scan_remembered_set(true /* is_concurrent */); } // Concurrent mark roots diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index 6630b80226ff1..2d3f27f70f1a6 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -781,7 +781,7 @@ ShenandoahObjToScanQueueSet* ShenandoahGeneration::old_gen_task_queues() const { return nullptr; } -void ShenandoahGeneration::scan_remembered_set() { +void ShenandoahGeneration::scan_remembered_set(bool is_concurrent) { assert(generation_mode() == YOUNG, "Should only scan remembered set for young generation."); ShenandoahHeap* const heap = ShenandoahHeap::heap(); @@ -789,8 +789,8 @@ void ShenandoahGeneration::scan_remembered_set() { reserve_task_queues(nworkers); ShenandoahReferenceProcessor* rp = ref_processor(); - ShenandoahRegionIterator regions; - ShenandoahScanRememberedTask task(task_queues(), old_gen_task_queues(), rp, ®ions); + ShenandoahRegionChunkIterator work_list(nworkers); + ShenandoahScanRememberedTask task(task_queues(), old_gen_task_queues(), rp, &work_list, is_concurrent); heap->workers()->run_task(&task); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index bc0907152ee39..9b69bcfdf6d7b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -160,7 +160,7 @@ class ShenandoahGeneration : public CHeapObj { virtual ShenandoahObjToScanQueueSet* old_gen_task_queues() const; // Scan remembered set at start of concurrent young-gen marking. */ - void scan_remembered_set(); + void scan_remembered_set(bool is_concurrent); void increment_affiliated_region_count(); void decrement_affiliated_region_count(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index b5473037131ea..8fd5bdb003b9f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -2470,12 +2470,15 @@ class ShenandoahUpdateHeapRefsTask : public WorkerTask { private: ShenandoahHeap* _heap; ShenandoahRegionIterator* _regions; + ShenandoahRegionChunkIterator* _work_chunks; public: - explicit ShenandoahUpdateHeapRefsTask(ShenandoahRegionIterator* regions) : + explicit ShenandoahUpdateHeapRefsTask(ShenandoahRegionIterator* regions, + ShenandoahRegionChunkIterator* work_chunks) : WorkerTask("Shenandoah Update References"), _heap(ShenandoahHeap::heap()), - _regions(regions) + _regions(regions), + _work_chunks(work_chunks) { } @@ -2504,60 +2507,24 @@ class ShenandoahUpdateHeapRefsTask : public WorkerTask { assert (update_watermark >= r->bottom(), "sanity"); log_debug(gc)("ShenandoahUpdateHeapRefsTask::do_work(%u) looking at region " SIZE_FORMAT, worker_id, r->index()); + bool region_progress = false; if (r->is_active() && !r->is_cset()) { if (!_heap->mode()->is_generational() || (r->affiliation() == ShenandoahRegionAffiliation::YOUNG_GENERATION)) { _heap->marked_object_oop_iterate(r, &cl, update_watermark); + region_progress = true; } else if (r->affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) { if (_heap->active_generation()->generation_mode() == GLOBAL) { + // Note that GLOBAL collection is not as effectively balanced as young and mixed cycles. This is because + // concurrent GC threads are parceled out entire heap regions of work at a time and there + // is no "catchup phase" consisting of remembered set scanning, during which parcels of work are smaller + // and more easily distributed more fairly across threads. + + // TODO: Consider an improvement to load balance GLOBAL GC. _heap->marked_object_oop_iterate(r, &cl, update_watermark); - } else { - // Old region in a young cycle or mixed cycle. - if (!is_mixed) { - // This is a young evac.. - _heap->card_scan()->process_region(r, &cl, true); - } else { - // This is a _mixed_evac. - // - // TODO: For _mixed_evac, consider building an old-gen remembered set that allows restricted updating - // within old-gen HeapRegions. This remembered set can be constructed by old-gen concurrent marking - // and augmented by card marking. For example, old-gen concurrent marking can remember for each old-gen - // card which other old-gen regions it refers to: none, one-other specifically, multiple-other non-specific. - // Update-references when _mixed_evac processess each old-gen memory range that has a traditional DIRTY - // card or if the "old-gen remembered set" indicates that this card holds pointers specifically to an - // old-gen region in the most recent collection set, or if this card holds pointers to other non-specific - // old-gen heap regions. - if (r->is_humongous()) { - // Need to examine both dirty and clean cards during mixed evac. - r->oop_iterate_humongous(&cl); - } else { - // This is a mixed evacuation. Old regions that are candidates for collection have not been coalesced - // and filled. Use mark bits to find objects that need to be updated. - // - // Future TODO: establish a second remembered set to identify which old-gen regions point to other old-gen - // regions which are in the collection set for a particular mixed evacuation. - HeapWord *p = r->bottom(); - ShenandoahObjectToOopBoundedClosure objs(&cl, p, update_watermark); - - // Anything beyond update_watermark was allocated during evacuation. Thus, it is known to not hold - // references to collection set objects. - while (p < update_watermark) { - oop obj = cast_to_oop(p); - if (ctx->is_marked(obj)) { - objs.do_object(obj); - p += obj->size(); - } else { - // This object is not marked so we don't scan it. - HeapWord* tams = ctx->top_at_mark_start(r); - if (p >= tams) { - p += obj->size(); - } else { - p = ctx->get_next_marked_addr(p, tams); - } - } - } - } - } + region_progress = true; } + // Otherwise, this is an old region in a young or mixed cycle. Process it during a second phase, below. + // Don't bother to report pacing progress in this case. } else { // Because updating of references runs concurrently, it is possible that a FREE inactive region transitions // to a non-free active region while this loop is executing. Whenever this happens, the changing of a region's @@ -2574,7 +2541,7 @@ class ShenandoahUpdateHeapRefsTask : public WorkerTask { affiliation_name(r->affiliation()), r->index()); } } - if (ShenandoahPacing) { + if (region_progress && ShenandoahPacing) { _heap->pacer()->report_updaterefs(pointer_delta(update_watermark, r->bottom())); } if (_heap->check_cancelled_gc_and_yield(CONCURRENT)) { @@ -2582,22 +2549,141 @@ class ShenandoahUpdateHeapRefsTask : public WorkerTask { } r = _regions->next(); } + if (_heap->mode()->is_generational() && (_heap->active_generation()->generation_mode() != GLOBAL)) { + // Since this is generational and not GLOBAL, we have to process the remembered set. There's no remembered + // set processing if not in generational mode or if GLOBAL mode. + + // After this thread has exhausted its traditional update-refs work, it continues with updating refs within remembered set. + // The remembered set workload is better balanced between threads, so threads that are "behind" can catch up with other + // threads during this phase, allowing all threads to work more effectively in parallel. + struct ShenandoahRegionChunk assignment; + bool have_work = _work_chunks->next(&assignment); + RememberedScanner* scanner = _heap->card_scan(); + while (have_work) { + ShenandoahHeapRegion* r = assignment._r; + if (r->is_active() && !r->is_cset() && (r->affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION)) { + HeapWord* start_of_range = r->bottom() + assignment._chunk_offset; + HeapWord* end_of_range = r->get_update_watermark(); + if (end_of_range > start_of_range + assignment._chunk_size) { + end_of_range = start_of_range + assignment._chunk_size; + } + + // Old region in a young cycle or mixed cycle. + if (is_mixed) { + // TODO: For mixed evac, consider building an old-gen remembered set that allows restricted updating + // within old-gen HeapRegions. This remembered set can be constructed by old-gen concurrent marking + // and augmented by card marking. For example, old-gen concurrent marking can remember for each old-gen + // card which other old-gen regions it refers to: none, one-other specifically, multiple-other non-specific. + // Update-references when _mixed_evac processess each old-gen memory range that has a traditional DIRTY + // card or if the "old-gen remembered set" indicates that this card holds pointers specifically to an + // old-gen region in the most recent collection set, or if this card holds pointers to other non-specific + // old-gen heap regions. + + if (r->is_humongous()) { + if (start_of_range < end_of_range) { + // Need to examine both dirty and clean cards during mixed evac. + r->oop_iterate_humongous_slice(&cl, false, start_of_range, assignment._chunk_size, true, CONCURRENT); + } + } else { + // Since this is mixed evacuation, old regions that are candidates for collection have not been coalesced + // and filled. Use mark bits to find objects that need to be updated. + // + // Future TODO: establish a second remembered set to identify which old-gen regions point to other old-gen + // regions which are in the collection set for a particular mixed evacuation. + if (start_of_range < end_of_range) { + HeapWord* p = nullptr; + size_t card_index = scanner->card_index_for_addr(start_of_range); + // In case last object in my range spans boundary of my chunk, I may need to scan all the way to top() + ShenandoahObjectToOopBoundedClosure objs(&cl, start_of_range, r->top()); + + // Any object that begins in a previous range is part of a different scanning assignment. Any object that + // starts after end_of_range is also not my responsibility. (Either allocated during evacuation, so does + // not hold pointers to from-space, or is beyond the range of my assigned work chunk.) + + // Find the first object that begins in my range, if there is one. + p = start_of_range; + oop obj = cast_to_oop(p); + HeapWord* tams = ctx->top_at_mark_start(r); + if (p >= tams) { + // We cannot use ctx->is_marked(obj) to test whether an object begins at this address. Instead, + // we need to use the remembered set crossing map to advance p to the first object that starts + // within the enclosing card. + + while (true) { + HeapWord* first_object = scanner->first_object_in_card(card_index); + if (first_object != nullptr) { + p = first_object; + break; + } else if (scanner->addr_for_card_index(card_index + 1) < end_of_range) { + card_index++; + } else { + // Force the loop that follows to immediately terminate. + p = end_of_range; + break; + } + } + obj = cast_to_oop(p); + // Note: p may be >= end_of_range + } else if (!ctx->is_marked(obj)) { + p = ctx->get_next_marked_addr(p, tams); + obj = cast_to_oop(p); + // If there are no more marked objects before tams, this returns tams. + // Note that tams is either >= end_of_range, or tams is the start of an object that is marked. + } + while (p < end_of_range) { + // p is known to point to the beginning of marked object obj + objs.do_object(obj); + HeapWord* prev_p = p; + p += obj->size(); + if (p < tams) { + p = ctx->get_next_marked_addr(p, tams); + // If there are no more marked objects before tams, this returns tams. Note that tams is + // either >= end_of_range, or tams is the start of an object that is marked. + } + assert(p != prev_p, "Lack of forward progress"); + obj = cast_to_oop(p); + } + } + } + } else { + // This is a young evac.. + if (start_of_range < end_of_range) { + size_t cluster_size = + CardTable::card_size_in_words() * ShenandoahCardCluster::CardsPerCluster; + size_t clusters = assignment._chunk_size / cluster_size; + assert(clusters * cluster_size == assignment._chunk_size, "Chunk assignment must align on cluster boundaries"); + scanner->process_region_slice(r, assignment._chunk_offset, clusters, end_of_range, &cl, true, CONCURRENT); + } + } + if (ShenandoahPacing && (start_of_range < end_of_range)) { + _heap->pacer()->report_updaterefs(pointer_delta(end_of_range, start_of_range)); + } + } + // Otherwise, this work chunk had nothing for me to do, so do not report pacer progress. + + // Before we take responsibility for another chunk of work, see if cancellation is requested. + if (_heap->check_cancelled_gc_and_yield(CONCURRENT)) { + return; + } + have_work = _work_chunks->next(&assignment); + } + } } }; void ShenandoahHeap::update_heap_references(bool concurrent) { assert(!is_full_gc_in_progress(), "Only for concurrent and degenerated GC"); + ShenandoahRegionChunkIterator work_list(workers()->active_workers()); if (concurrent) { - ShenandoahUpdateHeapRefsTask task(&_update_refs_iterator); + ShenandoahUpdateHeapRefsTask task(&_update_refs_iterator, &work_list); workers()->run_task(&task); } else { - ShenandoahUpdateHeapRefsTask task(&_update_refs_iterator); + ShenandoahUpdateHeapRefsTask task(&_update_refs_iterator, &work_list); workers()->run_task(&task); } } - class ShenandoahFinalUpdateRefsUpdateRegionStateClosure : public ShenandoahHeapRegionClosure { private: ShenandoahMarkingContext* _ctx; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp index ba29f303fcacb..37784d2061e6b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp @@ -116,7 +116,7 @@ void ShenandoahSTWMark::mark() { // Mark if (_generation->generation_mode() == YOUNG) { // But only scan the remembered set for young generation. - _generation->scan_remembered_set(); + _generation->scan_remembered_set(false /* is_concurrent */); } StrongRootsScope scope(nworkers); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp index 529b9ab6fbce3..2b2f467478af7 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp @@ -87,29 +87,200 @@ void ShenandoahDirectCardMarkRememberedSet::merge_overreach(size_t first_cluster ShenandoahScanRememberedTask::ShenandoahScanRememberedTask(ShenandoahObjToScanQueueSet* queue_set, ShenandoahObjToScanQueueSet* old_queue_set, ShenandoahReferenceProcessor* rp, - ShenandoahRegionIterator* regions) : + ShenandoahRegionChunkIterator* work_list, bool is_concurrent) : WorkerTask("Scan Remembered Set"), - _queue_set(queue_set), _old_queue_set(old_queue_set), _rp(rp), _regions(regions) {} + _queue_set(queue_set), _old_queue_set(old_queue_set), _rp(rp), _work_list(work_list), _is_concurrent(is_concurrent) {} void ShenandoahScanRememberedTask::work(uint worker_id) { - // This sets up a thread local reference to the worker_id which is necessary - // the weak reference processor. - ShenandoahParallelWorkerSession worker_session(worker_id); + if (_is_concurrent) { + // This sets up a thread local reference to the worker_id which is needed by the weak reference processor. + ShenandoahConcurrentWorkerSession worker_session(worker_id); + ShenandoahSuspendibleThreadSetJoiner stsj(ShenandoahSuspendibleWorkers); + do_work(worker_id); + } else { + // This sets up a thread local reference to the worker_id which is needed by the weak reference processor. + ShenandoahParallelWorkerSession worker_session(worker_id); + do_work(worker_id); + } +} + +void ShenandoahScanRememberedTask::do_work(uint worker_id) { ShenandoahWorkerTimingsTracker x(ShenandoahPhaseTimings::init_scan_rset, ShenandoahPhaseTimings::ScanClusters, worker_id); ShenandoahObjToScanQueue* q = _queue_set->queue(worker_id); ShenandoahObjToScanQueue* old = _old_queue_set == NULL ? NULL : _old_queue_set->queue(worker_id); ShenandoahMarkRefsClosure cl(q, _rp, old); - RememberedScanner* scanner = ShenandoahHeap::heap()->card_scan(); + ShenandoahHeap* heap = ShenandoahHeap::heap(); + RememberedScanner* scanner = heap->card_scan(); // set up thread local closure for shen ref processor _rp->set_mark_closure(worker_id, &cl); - ShenandoahHeapRegion* region = _regions->next(); - while (region != NULL) { - log_debug(gc)("ShenandoahScanRememberedTask::work(%u), looking at region " SIZE_FORMAT, worker_id, region->index()); + struct ShenandoahRegionChunk assignment; + bool has_work = _work_list->next(&assignment); + while (has_work) { +#ifdef ENABLE_REMEMBERED_SET_CANCELLATION + // This check is currently disabled to avoid crashes that occur + // when we try to cancel remembered set scanning + if (heap->check_cancelled_gc_and_yield(_is_concurrent)) { + return; + } +#endif + ShenandoahHeapRegion* region = assignment._r; + log_debug(gc)("ShenandoahScanRememberedTask::do_work(%u), processing slice of region " + SIZE_FORMAT " at offset " SIZE_FORMAT ", size: " SIZE_FORMAT, + worker_id, region->index(), assignment._chunk_offset, assignment._chunk_size); if (region->affiliation() == OLD_GENERATION) { - scanner->process_region(region, &cl); + size_t cluster_size = + CardTable::card_size_in_words() * ShenandoahCardCluster::CardsPerCluster; + size_t clusters = assignment._chunk_size / cluster_size; + assert(clusters * cluster_size == assignment._chunk_size, "Chunk assignments must align on cluster boundaries"); + HeapWord* end_of_range = region->bottom() + assignment._chunk_offset + assignment._chunk_size; + + // During concurrent mark, region->top() equals TAMS with respect to the current young-gen pass. */ + if (end_of_range > region->top()) { + end_of_range = region->top(); + } + scanner->process_region_slice(region, assignment._chunk_offset, clusters, end_of_range, &cl, false, _is_concurrent); + } + has_work = _work_list->next(&assignment); + } +} + +size_t ShenandoahRegionChunkIterator::calc_group_size() { + // The group size s calculated from the number of regions. Every group except the last processes the same number of chunks. + // The last group processes however many chunks are required to finish the total scanning effort. The chunk sizes are + // different for each group. The intention is that the first group processes roughly half of the heap, the second processes + // a quarter of the remaining heap, the third processes an eight of what remains and so on. The smallest chunk size + // is represented by _smallest_chunk_size. We do not divide work any smaller than this. + // + // Note that N/2 + N/4 + N/8 + N/16 + ... sums to N if expanded to infinite terms. + return _heap->num_regions() / 2; +} + +size_t ShenandoahRegionChunkIterator::calc_first_group_chunk_size() { + size_t words_in_region = ShenandoahHeapRegion::region_size_words(); + return words_in_region; +} + +size_t ShenandoahRegionChunkIterator::calc_num_groups() { + size_t total_heap_size = _heap->num_regions() * ShenandoahHeapRegion::region_size_words(); + size_t num_groups = 0; + size_t cumulative_group_span = 0; + size_t current_group_span = _first_group_chunk_size * _group_size; + size_t smallest_group_span = _smallest_chunk_size * _group_size; + while ((num_groups < _maximum_groups) && (cumulative_group_span + current_group_span <= total_heap_size)) { + num_groups++; + cumulative_group_span += current_group_span; + if (current_group_span <= smallest_group_span) { + break; + } else { + current_group_span /= 2; // Each group spans half of what the preceding group spanned. + } + } + // Loop post condition: + // num_groups <= _maximum_groups + // cumulative_group_span is the memory spanned by num_groups + // current_group_span is the span of the last fully populated group (assuming loop iterates at least once) + // each of num_groups is fully populated with _group_size chunks in each + // Non post conditions: + // cumulative_group_span may be less than total_heap size for one or more of the folowing reasons + // a) The number of regions remaining to be spanned is smaller than a complete group, or + // b) We have filled up all groups through _maximum_groups and still have not spanned all regions + + if (cumulative_group_span < total_heap_size) { + // We've got more regions to span + if ((num_groups < _maximum_groups) && (current_group_span > smallest_group_span)) { + num_groups++; // Place all remaining regions into a new not-full group (chunk_size half that of previous group) } - region = _regions->next(); + // Else we are unable to create a new group because we've exceed the number of allowed groups or have reached the + // minimum chunk size. + + // Any remaining regions will be treated as if they are part of the most recently created group. This group will + // have more than _group_size chunks within it. + } + return num_groups; +} + +size_t ShenandoahRegionChunkIterator::calc_total_chunks() { + size_t region_size_words = ShenandoahHeapRegion::region_size_words(); + size_t unspanned_heap_size = _heap->num_regions() * region_size_words; + size_t num_chunks = 0; + size_t spanned_groups = 0; + size_t cumulative_group_span = 0; + size_t current_group_span = _first_group_chunk_size * _group_size; + size_t smallest_group_span = _smallest_chunk_size * _group_size; + while (unspanned_heap_size > 0) { + if (current_group_span <= unspanned_heap_size) { + unspanned_heap_size -= current_group_span; + num_chunks += _group_size; + spanned_groups++; + + // _num_groups is the number of groups required to span the configured heap size. We are not allowed + // to change the number of groups. The last group is responsible for spanning all chunks not spanned + // by previously processed groups. + if (spanned_groups >= _num_groups) { + // The last group has more than _group_size entries. + size_t chunk_span = current_group_span / _group_size; + size_t extra_chunks = unspanned_heap_size / chunk_span; + assert (extra_chunks * chunk_span == unspanned_heap_size, "Chunks must precisely span regions"); + num_chunks += extra_chunks; + return num_chunks; + } else if (current_group_span <= smallest_group_span) { + // We cannot introduce new groups because we've reached the lower bound on group size + size_t chunk_span = _smallest_chunk_size; + size_t extra_chunks = unspanned_heap_size / chunk_span; + assert (extra_chunks * chunk_span == unspanned_heap_size, "Chunks must precisely span regions"); + num_chunks += extra_chunks; + return num_chunks; + } else { + current_group_span /= 2; + } + } else { + // The last group has fewer than _group_size entries. + size_t chunk_span = current_group_span / _group_size; + size_t last_group_size = unspanned_heap_size / chunk_span; + assert (last_group_size * chunk_span == unspanned_heap_size, "Chunks must precisely span regions"); + num_chunks += last_group_size; + return num_chunks; + } + } + return num_chunks; +} + +ShenandoahRegionChunkIterator::ShenandoahRegionChunkIterator(size_t worker_count) : + ShenandoahRegionChunkIterator(ShenandoahHeap::heap(), worker_count) +{ +} + +ShenandoahRegionChunkIterator::ShenandoahRegionChunkIterator(ShenandoahHeap* heap, size_t worker_count) : + _heap(heap), + _group_size(calc_group_size()), + _first_group_chunk_size(calc_first_group_chunk_size()), + _num_groups(calc_num_groups()), + _total_chunks(calc_total_chunks()), + _index(0) +{ + assert(_smallest_chunk_size == + CardTable::card_size_in_words() * ShenandoahCardCluster::CardsPerCluster, + "_smallest_chunk_size is not valid"); + + size_t words_in_region = ShenandoahHeapRegion::region_size_words(); + size_t group_span = _first_group_chunk_size * _group_size; + + _region_index[0] = 0; + _group_offset[0] = 0; + for (size_t i = 1; i < _num_groups; i++) { + _region_index[i] = _region_index[i-1] + (_group_offset[i-1] + group_span) / words_in_region; + _group_offset[i] = (_group_offset[i-1] + group_span) % words_in_region; + group_span /= 2; + } + // Not necessary, but keeps things tidy + for (size_t i = _num_groups; i < _maximum_groups; i++) { + _region_index[i] = 0; + _group_offset[i] = 0; } } + +void ShenandoahRegionChunkIterator::reset() { + _index = 0; +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp index f5c09940825df..aa56f9ea23658 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp @@ -209,6 +209,7 @@ #include "memory/iterator.hpp" #include "gc/shared/workerThread.hpp" #include "gc/shenandoah/shenandoahCardTable.hpp" +#include "gc/shenandoah/shenandoahHeap.hpp" #include "gc/shenandoah/shenandoahHeapRegion.hpp" #include "gc/shenandoah/shenandoahTaskqueue.hpp" @@ -923,12 +924,21 @@ class ShenandoahScanRemembered: public CHeapObj { void clear_old_remset() { _rs->clear_old_remset(); } size_t cluster_for_addr(HeapWord *addr); + HeapWord* addr_for_cluster(size_t cluster_no); void reset_object_range(HeapWord *from, HeapWord *to); void register_object(HeapWord *addr); void register_object_wo_lock(HeapWord *addr); void coalesce_objects(HeapWord *addr, size_t length_in_words); + HeapWord* first_object_in_card(size_t card_index) { + if (_scc->has_object(card_index)) { + return addr_for_card_index(card_index) + _scc->get_first_start(card_index); + } else { + return nullptr; + } + } + // Return true iff this object is "properly" registered. bool verify_registration(HeapWord* address, ShenandoahMarkingContext* ctx); @@ -967,16 +977,26 @@ class ShenandoahScanRemembered: public CHeapObj { // the template expansions were making it difficult for the link/loader to resolve references to the template- // parameterized implementations of this service. template - inline void process_clusters(size_t first_cluster, size_t count, HeapWord *end_of_range, ClosureType *oops); + inline void process_clusters(size_t first_cluster, size_t count, HeapWord *end_of_range, ClosureType *oops, bool is_concurrent); + + template + inline void process_clusters(size_t first_cluster, size_t count, HeapWord *end_of_range, ClosureType *oops, + bool use_write_table, bool is_concurrent); + + template + inline void process_humongous_clusters(ShenandoahHeapRegion* r, size_t first_cluster, size_t count, + HeapWord *end_of_range, ClosureType *oops, bool use_write_table, bool is_concurrent); + template - inline void process_clusters(size_t first_cluster, size_t count, HeapWord *end_of_range, ClosureType *oops, bool use_write_table); + inline void process_region(ShenandoahHeapRegion* region, ClosureType *cl, bool is_concurrent); template - inline void process_region(ShenandoahHeapRegion* region, ClosureType *cl); + inline void process_region(ShenandoahHeapRegion* region, ClosureType *cl, bool use_write_table, bool is_concurrent); template - inline void process_region(ShenandoahHeapRegion* region, ClosureType *cl, bool use_write_table); + inline void process_region_slice(ShenandoahHeapRegion* region, size_t offset, size_t clusters, HeapWord* end_of_range, + ClosureType *cl, bool use_write_table, bool is_concurrent); // To Do: // Create subclasses of ShenandoahInitMarkRootsClosure and @@ -1002,6 +1022,80 @@ class ShenandoahScanRemembered: public CHeapObj { void roots_do(OopIterateClosure* cl); }; +struct ShenandoahRegionChunk { + ShenandoahHeapRegion *_r; + size_t _chunk_offset; // HeapWordSize offset + size_t _chunk_size; // HeapWordSize qty +}; + +class ShenandoahRegionChunkIterator : public StackObj { +private: + // smallest_chunk_size is 64 words per card * + // ShenandoahCardCluster::CardsPerCluster. + // This is computed from CardTable::card_size_in_words() * + // ShenandoahCardCluster::CardsPerCluster; + // We can't perform this computation here, because of encapsulation and initialization constraints. We paste + // the magic number here, and assert that this number matches the intended computation in constructor. + static const size_t _smallest_chunk_size = 64 * ShenandoahCardCluster::CardsPerCluster; + + // The total remembered set scanning effort is divided into chunks of work that are assigned to individual worker tasks. + // The chunks of assigned work are divided into groups, where the size of each group (_group_size) is 4 * the number of + // worker tasks. All of the assignments within a group represent the same amount of memory to be scanned. Each of the + // assignments within the first group are of size _first_group_chunk_size (typically the ShenandoahHeapRegion size, but + // possibly smaller. Each of the assignments within each subsequent group are half the size of the assignments in the + // preceding group. The last group may be larger than the others. Because no group is allowed to have smaller assignments + // than _smallest_chunk_size, which is 32 KB. + + // Under normal circumstances, no configuration needs more than _maximum_groups (default value of 16). + + static const size_t _maximum_groups = 16; + + const ShenandoahHeap* _heap; + + const size_t _group_size; // Number of chunks in each group, equals worker_threads * 8 + const size_t _first_group_chunk_size; + const size_t _num_groups; // Number of groups in this configuration + const size_t _total_chunks; + + shenandoah_padding(0); + volatile size_t _index; + shenandoah_padding(1); + + size_t _region_index[_maximum_groups]; + size_t _group_offset[_maximum_groups]; + + + // No implicit copying: iterators should be passed by reference to capture the state + NONCOPYABLE(ShenandoahRegionChunkIterator); + + // Makes use of _heap. + size_t calc_group_size(); + + // Makes use of _group_size, which must be initialized before call. + size_t calc_first_group_chunk_size(); + + // Makes use of _group_size and _first_group_chunk_size, both of which must be initialized before call. + size_t calc_num_groups(); + + // Makes use of _group_size, _first_group_chunk_size, which must be initialized before call. + size_t calc_total_chunks(); + +public: + ShenandoahRegionChunkIterator(size_t worker_count); + ShenandoahRegionChunkIterator(ShenandoahHeap* heap, size_t worker_count); + + // Reset iterator to default state + void reset(); + + // Fills in assignment with next chunk of work and returns true iff there is more work. + // Otherwise, returns false. This is multi-thread-safe. + inline bool next(struct ShenandoahRegionChunk *assignment); + + // This is *not* MT safe. However, in the absence of multithreaded access, it + // can be used to determine if there is more work to do. + inline bool has_next() const; +}; + typedef ShenandoahScanRemembered RememberedScanner; class ShenandoahScanRememberedTask : public WorkerTask { @@ -1009,13 +1103,16 @@ class ShenandoahScanRememberedTask : public WorkerTask { ShenandoahObjToScanQueueSet* _queue_set; ShenandoahObjToScanQueueSet* _old_queue_set; ShenandoahReferenceProcessor* _rp; - ShenandoahRegionIterator* _regions; + ShenandoahRegionChunkIterator* _work_list; + bool _is_concurrent; public: ShenandoahScanRememberedTask(ShenandoahObjToScanQueueSet* queue_set, ShenandoahObjToScanQueueSet* old_queue_set, ShenandoahReferenceProcessor* rp, - ShenandoahRegionIterator* regions); + ShenandoahRegionChunkIterator* work_list, + bool is_concurrent); void work(uint worker_id); + void do_work(uint worker_id); }; #endif // SHARE_GC_SHENANDOAH_SHENANDOAHSCANREMEMBERED_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp index 3c69ff75bae20..5359def6cc105 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp @@ -449,9 +449,12 @@ ShenandoahScanRemembered::verify_registration(HeapWord* address, last_obj = obj; } else { offset = ctx->get_next_marked_addr(base_addr + offset, tams) - base_addr; - // offset will be zero if no objects are marked in this card. + // If there are no marked objects remaining in this region, offset equals tams - base_addr. If this offset is + // greater than max_offset, we will immediately exit this loop. Otherwise, the next iteration of the loop will + // treat the object at offset as marked and live (because address >= tams) and we will continue iterating object + // by consulting the size() fields of each. } - } while (offset > 0 && offset < max_offset); + } while (offset < max_offset); if (last_obj != nullptr && prev_offset + last_obj->size() >= max_offset) { // last marked object extends beyond end of card if (_scc->get_last_start(index) != prev_offset) { @@ -481,18 +484,21 @@ template template inline void ShenandoahScanRemembered::process_clusters(size_t first_cluster, size_t count, HeapWord *end_of_range, - ClosureType *cl) { - process_clusters(first_cluster, count, end_of_range, cl, false); + ClosureType *cl, bool is_concurrent) { + process_clusters(first_cluster, count, end_of_range, cl, false, is_concurrent); } // Process all objects starting within count clusters beginning with first_cluster for which the start address is -// less than end_of_range. For any such object, process the complete object, even if its end reaches beyond -// end_of_range. +// less than end_of_range. For any such object, process the complete object, even if its end reaches beyond end_of_range. + +// Do not CANCEL within process_clusters. It is assumed that if a worker thread accepts responsbility for processing +// a chunk of work, it will finish the work it starts. Otherwise, the chunk of work will be lost in the transition to +// degenerated execution. template template inline void ShenandoahScanRemembered::process_clusters(size_t first_cluster, size_t count, HeapWord *end_of_range, - ClosureType *cl, bool write_table) { + ClosureType *cl, bool write_table, bool is_concurrent) { // Unlike traditional Shenandoah marking, the old-gen resident objects that are examined as part of the remembered set are not // themselves marked. Each such object will be scanned only once. Any young-gen objects referenced from the remembered set will @@ -514,14 +520,24 @@ ShenandoahScanRemembered::process_clusters(size_t first_cluster, ctx = nullptr; } - HeapWord* end_of_clusters = _rs->addr_for_card_index(first_cluster) - + count * ShenandoahCardCluster::CardsPerCluster * CardTable::card_size_in_words(); + size_t card_index = first_cluster * ShenandoahCardCluster::CardsPerCluster; + HeapWord *start_of_range = _rs->addr_for_card_index(card_index); + ShenandoahHeapRegion* r = heap->heap_region_containing(start_of_range); + assert(end_of_range <= r->top(), "process_clusters() examines one region at a time"); + while (count-- > 0) { - size_t card_index = first_cluster * ShenandoahCardCluster::CardsPerCluster; + // TODO: do we want to check cancellation in inner loop, on every card processed? That would be more responsive, + // but require more overhead for checking. + card_index = first_cluster * ShenandoahCardCluster::CardsPerCluster; size_t end_card_index = card_index + ShenandoahCardCluster::CardsPerCluster; first_cluster++; size_t next_card_index = 0; while (card_index < end_card_index) { + if (_rs->addr_for_card_index(card_index) > end_of_range) { + count = 0; + card_index = end_card_index; + break; + } bool is_dirty = (write_table)? is_write_card_dirty(card_index): is_card_dirty(card_index); bool has_object = _scc->has_object(card_index); if (is_dirty) { @@ -532,6 +548,8 @@ ShenandoahScanRemembered::process_clusters(size_t first_cluster, HeapWord *p = _rs->addr_for_card_index(card_index); HeapWord *card_start = p; HeapWord *endp = p + CardTable::card_size_in_words(); + assert(!r->is_humongous(), "Process humongous regions elsewhere"); + if (endp > end_of_range) { endp = end_of_range; next_card_index = end_card_index; @@ -569,8 +587,7 @@ ShenandoahScanRemembered::process_clusters(size_t first_cluster, } p += obj->size(); } else { - // This object is not marked so we don't scan it. - ShenandoahHeapRegion* r = heap->heap_region_containing(p); + // This object is not marked so we don't scan it. Containing region r is initialized above. HeapWord* tams = ctx->top_at_mark_start(r); if (p >= tams) { p += obj->size(); @@ -618,6 +635,11 @@ ShenandoahScanRemembered::process_clusters(size_t first_cluster, } } + // TODO: only iterate over this object if it spans dirty within this cluster or within following clusters. + // Code as written is known not to examine a zombie object because either the object is marked, or we are + // not using the mark-context to differentiate objects, so the object is known to have been coalesced and + // filled if it is not "live". + if (reaches_next_cluster || spans_dirty_within_this_cluster) { if (obj->is_objArray()) { objArrayOop array = objArrayOop(obj); @@ -635,8 +657,7 @@ ShenandoahScanRemembered::process_clusters(size_t first_cluster, } } else { // The object that spans end of this clean card is not marked, so no need to scan it or its - // unmarked neighbors. - ShenandoahHeapRegion* r = heap->heap_region_containing(p); + // unmarked neighbors. Containing region r is initialized above. HeapWord* tams = ctx->top_at_mark_start(r); HeapWord* nextp; if (p >= tams) { @@ -656,19 +677,58 @@ ShenandoahScanRemembered::process_clusters(size_t first_cluster, } } +// Given that this range of clusters is known to span a humongous object spanned by region r, scan the +// portion of the humongous object that corresponds to the specified range. template template inline void -ShenandoahScanRemembered::process_region(ShenandoahHeapRegion *region, ClosureType *cl) { - process_region(region, cl, false); +ShenandoahScanRemembered::process_humongous_clusters(ShenandoahHeapRegion* r, size_t first_cluster, size_t count, + HeapWord *end_of_range, ClosureType *cl, bool write_table, + bool is_concurrent) { + ShenandoahHeapRegion* start_region = r->humongous_start_region(); + HeapWord* p = start_region->bottom(); + oop obj = cast_to_oop(p); + assert(r->is_humongous(), "Only process humongous regions here"); + assert(start_region->is_humongous_start(), "Should be start of humongous region"); + assert(p + obj->size() >= end_of_range, "Humongous object ends before range ends"); + + size_t first_card_index = first_cluster * ShenandoahCardCluster::CardsPerCluster; + HeapWord* first_cluster_addr = _rs->addr_for_card_index(first_card_index); + size_t spanned_words = count * ShenandoahCardCluster::CardsPerCluster * CardTable::card_size_in_words(); + + start_region->oop_iterate_humongous_slice(cl, true, first_cluster_addr, spanned_words, write_table, is_concurrent); } template template inline void -ShenandoahScanRemembered::process_region(ShenandoahHeapRegion *region, ClosureType *cl, bool use_write_table) { - HeapWord *start_of_range = region->bottom(); +ShenandoahScanRemembered::process_region(ShenandoahHeapRegion *region, ClosureType *cl, bool is_concurrent) { + process_region(region, cl, false, is_concurrent); +} + +template +template +inline void +ShenandoahScanRemembered::process_region(ShenandoahHeapRegion *region, ClosureType *cl, + bool use_write_table, bool is_concurrent) { + size_t cluster_size = + CardTable::card_size_in_words() * ShenandoahCardCluster::CardsPerCluster; + size_t clusters = ShenandoahHeapRegion::region_size_words() / cluster_size; + process_region_slice(region, 0, clusters, region->end(), cl, use_write_table, is_concurrent); +} + +template +template +inline void +ShenandoahScanRemembered::process_region_slice(ShenandoahHeapRegion *region, size_t start_offset, size_t clusters, + HeapWord *end_of_range, ClosureType *cl, bool use_write_table, + bool is_concurrent) { + HeapWord *start_of_range = region->bottom() + start_offset; + size_t cluster_size = + CardTable::card_size_in_words() * ShenandoahCardCluster::CardsPerCluster; + size_t words = clusters * cluster_size; size_t start_cluster_no = cluster_for_addr(start_of_range); + assert(addr_for_cluster(start_cluster_no) == start_of_range, "process_region_slice range must align on cluster boundary"); // region->end() represents the end of memory spanned by this region, but not all of this // memory is eligible to be scanned because some of this memory has not yet been allocated. @@ -676,34 +736,43 @@ ShenandoahScanRemembered::process_region(ShenandoahHeapRegion *re // region->top() represents the end of allocated memory within this region. Any addresses // beyond region->top() should not be scanned as that memory does not hold valid objects. - HeapWord *end_of_range; if (use_write_table) { // This is update-refs servicing. - end_of_range = region->get_update_watermark(); + if (end_of_range > region->get_update_watermark()) { + end_of_range = region->get_update_watermark(); + } } else { // This is concurrent mark servicing. Note that TAMS for this region is TAMS at start of old-gen // collection. Here, we need to scan up to TAMS for most recently initiated young-gen collection. // Since all LABs are retired at init mark, and since replacement LABs are allocated lazily, and since no // promotions occur until evacuation phase, TAMS for most recent young-gen is same as top(). - end_of_range = region->top(); + if (end_of_range > region->top()) { + end_of_range = region->top(); + } } log_debug(gc)("Remembered set scan processing Region " SIZE_FORMAT ", from " PTR_FORMAT " to " PTR_FORMAT ", using %s table", - region->index(), p2i(region->bottom()), p2i(end_of_range), + region->index(), p2i(start_of_range), p2i(end_of_range), use_write_table? "read/write (updating)": "read (marking)"); - // end_of_range may point to the middle of a cluster because region->top() may be different than region->end(). + + // Note that end_of_range may point to the middle of a cluster because region->top() or region->get_update_watermark() may + // be less than start_of_range + words. + // We want to assure that our process_clusters() request spans all relevant clusters. Note that each cluster // processed will avoid processing beyond end_of_range. // Note that any object that starts between start_of_range and end_of_range, including humongous objects, will // be fully processed by process_clusters, even though the object may reach beyond end_of_range. - size_t num_heapwords = end_of_range - start_of_range; - unsigned int cluster_size = CardTable::card_size_in_words() * ShenandoahCardCluster::CardsPerCluster; - size_t num_clusters = (size_t) ((num_heapwords - 1 + cluster_size) / cluster_size); - if (!region->is_humongous_continuation()) { - // Remembered set scanner - process_clusters(start_cluster_no, num_clusters, end_of_range, cl, use_write_table); + // If I am assigned to process a range that starts beyond end_of_range (top or update-watermark), we have no work to do. + + if (start_of_range < end_of_range) { + if (region->is_humongous()) { + ShenandoahHeapRegion* start_region = region->humongous_start_region(); + process_humongous_clusters(start_region, start_cluster_no, clusters, end_of_range, cl, use_write_table, is_concurrent); + } else { + process_clusters(start_cluster_no, clusters, end_of_range, cl, use_write_table, is_concurrent); + } } } @@ -715,6 +784,13 @@ ShenandoahScanRemembered::cluster_for_addr(HeapWordImpl **addr) { return result; } +template +inline HeapWord* +ShenandoahScanRemembered::addr_for_cluster(size_t cluster_no) { + size_t card_index = cluster_no * ShenandoahCardCluster::CardsPerCluster; + return addr_for_card_index(card_index); +} + // This is used only for debug verification so don't worry about making the scan parallel. template inline void ShenandoahScanRemembered::roots_do(OopIterateClosure* cl) { @@ -731,9 +807,53 @@ inline void ShenandoahScanRemembered::roots_do(OopIterateClosure* size_t num_clusters = (size_t) ((num_heapwords - 1 + cluster_size) / cluster_size); // Remembered set scanner - process_clusters(start_cluster_no, num_clusters, end_of_range, cl); + if (region->is_humongous()) { + process_humongous_clusters(region->humongous_start_region(), start_cluster_no, num_clusters, end_of_range, cl, + false /* is_write_table */, false /* is_concurrent */); + } else { + process_clusters(start_cluster_no, num_clusters, end_of_range, cl, false /* is_concurrent */); + } } } } +inline bool ShenandoahRegionChunkIterator::has_next() const { + return _index < _total_chunks; +} + +inline bool ShenandoahRegionChunkIterator::next(struct ShenandoahRegionChunk *assignment) { + if (_index > _total_chunks) { + return false; + } + size_t new_index = Atomic::add(&_index, (size_t) 1, memory_order_relaxed); + if (new_index > _total_chunks) { + return false; + } + // convert to zero-based indexing + new_index--; + + size_t group_no = new_index / _group_size; + if (group_no + 1 > _num_groups) { + group_no = _num_groups - 1; + } + + // All size computations measured in HeapWord + size_t region_size_words = ShenandoahHeapRegion::region_size_words(); + size_t group_region_index = _region_index[group_no]; + size_t group_region_offset = _group_offset[group_no]; + + size_t index_within_group = new_index - (group_no * _group_size); + size_t group_chunk_size = _first_group_chunk_size >> group_no; + size_t offset_of_this_chunk = group_region_offset + index_within_group * group_chunk_size; + size_t regions_spanned_by_chunk_offset = offset_of_this_chunk / region_size_words; + size_t region_index = group_region_index + regions_spanned_by_chunk_offset; + size_t offset_within_region = offset_of_this_chunk % region_size_words; + + assignment->_r = _heap->get_region(region_index); + assignment->_chunk_offset = offset_within_region; + assignment->_chunk_size = group_chunk_size; + + return true; +} + #endif // SHARE_GC_SHENANDOAH_SHENANDOAHSCANREMEMBEREDINLINE_HPP From afe4d0affc59fffca134fe8e41249e38b0071c80 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Tue, 9 Aug 2022 18:06:35 +0000 Subject: [PATCH 143/254] Include old humongous start regions when counting immediate garbage Reviewed-by: ysr, kdnilsen --- .../gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp index b3f127bf0400b..61dad8edcbc82 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp @@ -145,7 +145,9 @@ void ShenandoahOldHeuristics::prepare_for_old_collections() { // The humongous object is dead, we can just return this region and the continuations // immediately to the freeset - no evacuations are necessary here. The continuations // will be made into trash by this method, so they'll be skipped by the 'is_regular' - // check above. + // check above, but we still need to count the start region. + immediate_regions++; + immediate_garbage += garbage; size_t region_count = heap->trash_humongous_region_at(region); log_debug(gc)("Trashed " SIZE_FORMAT " regions for humongous object.", region_count); } From 6acf4e56d490fe956c5bdc2f72602295af8d6dd2 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Tue, 9 Aug 2022 18:13:45 +0000 Subject: [PATCH 144/254] Handle old, pinned regions Reviewed-by: ysr, kdnilsen --- .../heuristics/shenandoahHeuristics.cpp | 4 +- .../heuristics/shenandoahOldHeuristics.cpp | 208 ++++++++----- .../heuristics/shenandoahOldHeuristics.hpp | 94 +++--- .../gc/shenandoah/shenandoahConcurrentGC.cpp | 9 +- .../gc/shenandoah/shenandoahControlThread.cpp | 178 +++++++----- .../gc/shenandoah/shenandoahControlThread.hpp | 16 +- .../gc/shenandoah/shenandoahDegeneratedGC.cpp | 3 +- .../share/gc/shenandoah/shenandoahFreeSet.cpp | 1 + .../share/gc/shenandoah/shenandoahFullGC.cpp | 1 + .../gc/shenandoah/shenandoahGeneration.cpp | 41 +-- .../gc/shenandoah/shenandoahGeneration.hpp | 9 +- .../share/gc/shenandoah/shenandoahHeap.cpp | 34 ++- .../share/gc/shenandoah/shenandoahHeap.hpp | 16 +- .../gc/shenandoah/shenandoahHeap.inline.hpp | 4 + .../gc/shenandoah/shenandoahHeapRegion.cpp | 1 + .../gc/shenandoah/shenandoahInitLogger.cpp | 2 +- .../gc/shenandoah/shenandoahMarkClosures.cpp | 31 +- .../gc/shenandoah/shenandoahMarkClosures.hpp | 14 +- .../share/gc/shenandoah/shenandoahOldGC.cpp | 206 +++---------- .../share/gc/shenandoah/shenandoahOldGC.hpp | 5 - .../gc/shenandoah/shenandoahOldGeneration.cpp | 222 +++++++++++++- .../gc/shenandoah/shenandoahOldGeneration.hpp | 29 ++ .../gc/shenandoah/shenandoahPhaseTimings.hpp | 4 +- .../shenandoah/shenandoahRegulatorThread.cpp | 4 +- .../shenandoahScanRemembered.inline.hpp | 2 +- .../share/gc/shenandoah/shenandoahUtils.cpp | 2 +- .../gc/shenandoah/shenandoahVMOperations.hpp | 8 +- .../shenandoah/shenandoahYoungGeneration.cpp | 4 +- .../test_shenandoahOldHeuristic.cpp | 273 ++++++++++++++++++ 29 files changed, 976 insertions(+), 449 deletions(-) create mode 100644 test/hotspot/gtest/gc/shenandoah/test_shenandoahOldHeuristic.cpp diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp index d055124ac06e3..8422256a994b3 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp @@ -124,8 +124,6 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec size_t free_regions = 0; size_t live_memory = 0; - ShenandoahMarkingContext* const ctx = _generation->complete_marking_context(); - for (size_t i = 0; i < num_regions; i++) { ShenandoahHeapRegion* region = heap->get_region(i); if (!in_generation(region)) { @@ -160,7 +158,7 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec // Reclaim humongous regions here, and count them as the immediate garbage #ifdef ASSERT bool reg_live = region->has_live(); - bool bm_live = ctx->is_marked(cast_to_oop(region->bottom())); + bool bm_live = heap->complete_marking_context()->is_marked(cast_to_oop(region->bottom())); assert(reg_live == bm_live, "Humongous liveness and marks should agree. Region live: %s; Bitmap live: %s; Region Live Words: " SIZE_FORMAT, BOOL_TO_STR(reg_live), BOOL_TO_STR(bm_live), region->get_live_data_words()); diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp index 61dad8edcbc82..45d1a43d61680 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp @@ -25,22 +25,29 @@ #include "precompiled.hpp" #include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp" -#include "gc/shenandoah/shenandoahGeneration.hpp" +#include "gc/shenandoah/shenandoahCollectionSet.hpp" +#include "gc/shenandoah/shenandoahHeap.hpp" +#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" #include "gc/shenandoah/shenandoahOldGeneration.hpp" #include "gc/shenandoah/shenandoahYoungGeneration.hpp" #include "utilities/quickSort.hpp" -ShenandoahOldHeuristics::ShenandoahOldHeuristics(ShenandoahGeneration* generation, ShenandoahHeuristics* trigger_heuristic) : - ShenandoahHeuristics(generation), - _old_collection_candidates(0), - _next_old_collection_candidate(0), - _hidden_old_collection_candidates(0), - _hidden_next_old_collection_candidate(0), - _old_coalesce_and_fill_candidates(0), - _first_coalesce_and_fill_candidate(0), - _trigger_heuristic(trigger_heuristic), - _promotion_failed(false) +uint ShenandoahOldHeuristics::NOT_FOUND = -1U; + +ShenandoahOldHeuristics::ShenandoahOldHeuristics(ShenandoahOldGeneration* generation, ShenandoahHeuristics* trigger_heuristic) : + ShenandoahHeuristics(generation), +#ifdef ASSERT + _start_candidate(0), +#endif + _first_pinned_candidate(NOT_FOUND), + _last_old_collection_candidate(0), + _next_old_collection_candidate(0), + _last_old_region(0), + _trigger_heuristic(trigger_heuristic), + _promotion_failed(false), + _old_generation(generation) { + assert(_generation->generation_mode() == OLD, "This service only available for old-gc heuristics"); } bool ShenandoahOldHeuristics::prime_collection_set(ShenandoahCollectionSet* collection_set) { @@ -48,6 +55,8 @@ bool ShenandoahOldHeuristics::prime_collection_set(ShenandoahCollectionSet* coll return false; } + _first_pinned_candidate = NOT_FOUND; + ShenandoahHeap* heap = ShenandoahHeap::heap(); uint included_old_regions = 0; size_t evacuated_old_bytes = 0; @@ -58,7 +67,7 @@ bool ShenandoahOldHeuristics::prime_collection_set(ShenandoahCollectionSet* coll // of memory that can still be evacuated. We address this by reducing the evacuation budget by the amount // of live memory in that region and by the amount of unallocated memory in that region if the evacuation // budget is constrained by availability of free memory. - size_t old_evacuation_budget = (size_t) (heap->get_old_evac_reserve() / ShenandoahEvacWaste); + size_t old_evacuation_budget = (size_t) ((double) heap->get_old_evac_reserve() / ShenandoahEvacWaste); size_t remaining_old_evacuation_budget = old_evacuation_budget; size_t lost_evacuation_capacity = 0; log_info(gc)("Choose old regions for mixed collection: old evacuation budget: " SIZE_FORMAT "%s, candidates: %u", @@ -70,6 +79,9 @@ bool ShenandoahOldHeuristics::prime_collection_set(ShenandoahCollectionSet* coll while (unprocessed_old_collection_candidates() > 0) { // Old collection candidates are sorted in order of decreasing garbage contained therein. ShenandoahHeapRegion* r = next_old_collection_candidate(); + if (r == nullptr) { + break; + } // If we choose region r to be collected, then we need to decrease the capacity to hold other evacuations by // the size of r's free memory. @@ -90,15 +102,83 @@ bool ShenandoahOldHeuristics::prime_collection_set(ShenandoahCollectionSet* coll } } + if (_first_pinned_candidate != NOT_FOUND) { + // Need to deal with pinned regions + slide_pinned_regions_to_front(); + } + if (included_old_regions > 0) { log_info(gc)("Old-gen piggyback evac (" UINT32_FORMAT " regions, evacuating " SIZE_FORMAT "%s, reclaiming: " SIZE_FORMAT "%s)", included_old_regions, byte_size_in_proper_unit(evacuated_old_bytes), proper_unit_for_byte_size(evacuated_old_bytes), byte_size_in_proper_unit(collected_old_bytes), proper_unit_for_byte_size(collected_old_bytes)); } + + if (unprocessed_old_collection_candidates() == 0) { + _old_generation->transition_to(ShenandoahOldGeneration::IDLE); + } + return (included_old_regions > 0); } +void ShenandoahOldHeuristics::slide_pinned_regions_to_front() { + // Find the leftmost unpinned region. The region in this slot will have been + // added to the cset, so we can use it to hold pointers to regions that were + // pinned when the cset was chosen. + // [ r p r p p p r ] + // ^ + // | first r to the left should be in the collection set now. + uint write_index = NOT_FOUND; + for (uint search = _next_old_collection_candidate - 1; search > _first_pinned_candidate; --search) { + ShenandoahHeapRegion* region = _region_data[search]._region; + if (!region->is_pinned()) { + write_index = search; + assert(region->is_cset(), "Expected unpinned region to be added to the collection set."); + break; + } + } + + if (write_index == NOT_FOUND) { + if (_first_pinned_candidate > 0) { + _next_old_collection_candidate = _first_pinned_candidate; + } + return; + } + + // Find pinned regions to the left and move their pointer into a slot + // that was pointing at a region that has been added to the cset. + // [ r p r p p p r ] + // ^ + // | Write pointer is here. We know this region is already in the cset + // | so we can clobber it with the next pinned region we find. + for (size_t search = write_index - 1; search > _first_pinned_candidate; --search) { + RegionData& skipped = _region_data[search]; + if (skipped._region->is_pinned()) { + RegionData& added_to_cset = _region_data[write_index]; + assert(added_to_cset._region->is_cset(), "Can only overwrite slots used by regions added to the collection set."); + added_to_cset._region = skipped._region; + added_to_cset._garbage = skipped._garbage; + --write_index; + } + } + + // Everything left should already be in the cset + // [ r x p p p p r ] + // ^ + // | next pointer points at the first region which was not added + // | to the collection set. +#ifdef ASSERT + for (size_t check = write_index - 1; check > _start_candidate; --check) { + ShenandoahHeapRegion* region = _region_data[check]._region; + assert(region->is_cset(), "All regions here should be in the collection set."); + } + _start_candidate = write_index; +#endif + + // Update to read from the leftmost pinned region. + _next_old_collection_candidate = write_index; +} + // Both arguments are don't cares for old-gen collections void ShenandoahOldHeuristics::choose_collection_set(ShenandoahCollectionSet* collection_set, ShenandoahOldHeuristics* old_heuristics) { @@ -129,8 +209,9 @@ void ShenandoahOldHeuristics::prepare_for_old_collections() { size_t garbage = region->garbage(); total_garbage += garbage; - if (region->is_regular()) { + if (region->is_regular() || region->is_pinned()) { if (!region->has_live()) { + assert(!region->is_pinned(), "Pinned region should have live (pinned) objects."); region->make_trash_immediate(); immediate_regions++; immediate_garbage += garbage; @@ -170,93 +251,83 @@ void ShenandoahOldHeuristics::prepare_for_old_collections() { // // TODO: allow ShenandoahOldGarbageThreshold to be determined adaptively, by heuristics. + const size_t garbage_threshold = ShenandoahHeapRegion::region_size_bytes() * ShenandoahOldGarbageThreshold / 100; size_t candidates_garbage = 0; + _last_old_region = (uint)cand_idx; + _last_old_collection_candidate = (uint)cand_idx; + _next_old_collection_candidate = 0; + for (size_t i = 0; i < cand_idx; i++) { - candidates_garbage += candidates[i]._garbage; if (candidates[i]._garbage < garbage_threshold) { // Candidates are sorted in decreasing order of garbage, so no regions after this will be above the threshold - _hidden_next_old_collection_candidate = 0; - _hidden_old_collection_candidates = (uint)i; - _first_coalesce_and_fill_candidate = (uint)i; - _old_coalesce_and_fill_candidates = (uint)(cand_idx - i); - - // Note that we do not coalesce and fill occupied humongous regions - // HR: humongous regions, RR: regular regions, CF: coalesce and fill regions - log_info(gc)("Old-gen mark evac (" UINT32_FORMAT " RR, " UINT32_FORMAT " CF)", - _hidden_old_collection_candidates, _old_coalesce_and_fill_candidates); - return; + _last_old_collection_candidate = (uint)i; + break; } + candidates_garbage += candidates[i]._garbage; } - // If we reach here, all of non-humogous old-gen regions are candidates for collection set. - _hidden_next_old_collection_candidate = 0; - _hidden_old_collection_candidates = (uint)cand_idx; - _first_coalesce_and_fill_candidate = 0; - _old_coalesce_and_fill_candidates = 0; - // Note that we do not coalesce and fill occupied humongous regions // HR: humongous regions, RR: regular regions, CF: coalesce and fill regions size_t collectable_garbage = immediate_garbage + candidates_garbage; - log_info(gc)("Old-gen mark evac (" UINT32_FORMAT " RR, " UINT32_FORMAT " CF), " - "Collectable Garbage: " SIZE_FORMAT "%s, " - "Immediate Garbage: " SIZE_FORMAT "%s", - _hidden_old_collection_candidates, _old_coalesce_and_fill_candidates, - byte_size_in_proper_unit(collectable_garbage), proper_unit_for_byte_size(collectable_garbage), - byte_size_in_proper_unit(immediate_garbage), proper_unit_for_byte_size(immediate_garbage)); -} - -void ShenandoahOldHeuristics::start_old_evacuations() { - assert(_generation->generation_mode() == OLD, "This service only available for old-gc heuristics"); - - _old_collection_candidates = _hidden_old_collection_candidates; - _next_old_collection_candidate = _hidden_next_old_collection_candidate; + log_info(gc)("Old-Gen Collectable Garbage: " SIZE_FORMAT "%s over " UINT32_FORMAT " regions, " + "Old-Gen Immediate Garbage: " SIZE_FORMAT "%s over " SIZE_FORMAT " regions.", + byte_size_in_proper_unit(collectable_garbage), proper_unit_for_byte_size(collectable_garbage), _last_old_collection_candidate, + byte_size_in_proper_unit(immediate_garbage), proper_unit_for_byte_size(immediate_garbage), immediate_regions); - _hidden_old_collection_candidates = 0; + if (unprocessed_old_collection_candidates() == 0) { + _old_generation->transition_to(ShenandoahOldGeneration::IDLE); + } else { + _old_generation->transition_to(ShenandoahOldGeneration::WAITING); + } } -uint ShenandoahOldHeuristics::unprocessed_old_or_hidden_collection_candidates() { - assert(_generation->generation_mode() == OLD, "This service only available for old-gc heuristics"); - return _old_collection_candidates + _hidden_old_collection_candidates; +uint ShenandoahOldHeuristics::last_old_collection_candidate_index() { + return _last_old_collection_candidate; } uint ShenandoahOldHeuristics::unprocessed_old_collection_candidates() { - assert(_generation->generation_mode() == OLD, "This service only available for old-gc heuristics"); - return _old_collection_candidates; + return _last_old_collection_candidate - _next_old_collection_candidate; } ShenandoahHeapRegion* ShenandoahOldHeuristics::next_old_collection_candidate() { - assert(_generation->generation_mode() == OLD, "This service only available for old-gc heuristics"); - return _region_data[_next_old_collection_candidate]._region; + while (_next_old_collection_candidate < _last_old_collection_candidate) { + ShenandoahHeapRegion* next = _region_data[_next_old_collection_candidate]._region; + if (!next->is_pinned()) { + return next; + } else { + assert(next->is_pinned(), "sanity"); + if (_first_pinned_candidate == NOT_FOUND) { + _first_pinned_candidate = _next_old_collection_candidate; + } + } + + _next_old_collection_candidate++; + } + return nullptr; } void ShenandoahOldHeuristics::consume_old_collection_candidate() { - assert(_generation->generation_mode() == OLD, "This service only available for old-gc heuristics"); _next_old_collection_candidate++; - _old_collection_candidates--; } -uint ShenandoahOldHeuristics::old_coalesce_and_fill_candidates() { - assert(_generation->generation_mode() == OLD, "This service only available for old-gc heuristics"); - return _old_coalesce_and_fill_candidates; +uint ShenandoahOldHeuristics::last_old_region_index() const { + return _last_old_region; } -void ShenandoahOldHeuristics::get_coalesce_and_fill_candidates(ShenandoahHeapRegion** buffer) { - assert(_generation->generation_mode() == OLD, "This service only available for old-gc heuristics"); - uint count = _old_coalesce_and_fill_candidates; - int index = _first_coalesce_and_fill_candidate; - while (count-- > 0) { +unsigned int ShenandoahOldHeuristics::get_coalesce_and_fill_candidates(ShenandoahHeapRegion** buffer) { + uint end = _last_old_region; + uint index = _next_old_collection_candidate; + while (index < end) { *buffer++ = _region_data[index++]._region; } + return _last_old_region - _next_old_collection_candidate; } void ShenandoahOldHeuristics::abandon_collection_candidates() { - _old_collection_candidates = 0; + _last_old_collection_candidate = 0; _next_old_collection_candidate = 0; - _hidden_old_collection_candidates = 0; - _hidden_next_old_collection_candidate = 0; - _old_coalesce_and_fill_candidates = 0; - _first_coalesce_and_fill_candidate = 0; + _last_old_region = 0; } void ShenandoahOldHeuristics::handle_promotion_failure() { @@ -276,9 +347,8 @@ bool ShenandoahOldHeuristics::should_start_gc() { // Cannot start a new old-gen GC until previous one has finished. // // Future refinement: under certain circumstances, we might be more sophisticated about this choice. - // For example, we could choose to abandon the previous old collection before it has completed evacuations, - // but this would require that we coalesce and fill all garbage within unevacuated collection-set regions. - if (unprocessed_old_or_hidden_collection_candidates() > 0) { + // For example, we could choose to abandon the previous old collection before it has completed evacuations. + if (unprocessed_old_collection_candidates() > 0) { return false; } diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp index 2f77aa137692c..f4384f5aa43c1 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp @@ -25,74 +25,84 @@ #ifndef SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHOLDHEURISTICS_HPP #define SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHOLDHEURISTICS_HPP -#include "gc/shenandoah/shenandoahCollectionSet.inline.hpp" -#include "gc/shenandoah/shenandoahGeneration.hpp" -#include "gc/shenandoah/shenandoahHeap.inline.hpp" -#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" + #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" +class ShenandoahCollectionSet; +class ShenandoahHeapRegion; +class ShenandoahOldGeneration; + class ShenandoahOldHeuristics : public ShenandoahHeuristics { private: - // if (_generation->generation_mode() == OLD) _old_collection_candidates - // represent the number of regions selected for collection following the - // most recently completed old-gen mark that have not yet been selected - // for evacuation and _next_collection_candidate is the index within - // _region_data of the next candidate region to be selected for evacuation. - // if (_generation->generation_mode() != OLD) these two variables are - // not used. - uint _old_collection_candidates; + static uint NOT_FOUND; + + // After final marking of the old generation, this heuristic will select + // a set of candidate regions to be included in subsequent mixed collections. + // The regions are sorted into a `_region_data` array (declared in base + // class) in decreasing order of garbage. The heuristic will give priority + // to regions containing more garbage. + + // The following members are used to keep track of which candidate regions + // have yet to be added to a mixed collection. There is also some special + // handling for pinned regions, described further below. + + // This points to the first candidate of the current mixed collection. This + // is only used for an assertion when handling pinned regions. + debug_only(uint _start_candidate); + + // Pinned regions may not be included in the collection set. Any old regions + // which were pinned at the time when old regions were added to the mixed + // collection will have been skipped. These regions are still contain garbage, + // so we want to include them at the start of the list of candidates for the + // _next_ mixed collection cycle. This variable is the index of the _first_ + // old region which is pinned when the mixed collection set is formed. + uint _first_pinned_candidate; + + // This is the index of the last region which is above the garbage threshold. + // No regions after this will be considered for inclusion in a mixed collection + // set. + uint _last_old_collection_candidate; + + // This index points to the first candidate in line to be added to the mixed + // collection set. It is updated as regions are added to the collection set. uint _next_old_collection_candidate; - // At the time we select the old-gen collection set, _hidden_old_collection_candidates - // and _hidden_next_old_collection_candidates are set to remember the intended old-gen - // collection set. After all old-gen regions not in the old-gen collection set have been - // coalesced and filled, the content of these variables is copied to _old_collection_candidates - // and _next_old_collection_candidates so that evacuations can begin evacuating these regions. - uint _hidden_old_collection_candidates; - uint _hidden_next_old_collection_candidate; - - // if (_generation->generation_mode() == OLD) - // _old_coalesce_and_fill_candidates represents the number of regions - // that were chosen for the garbage contained therein to be coalesced - // and filled and _first_coalesce_and_fill_candidate represents the - // the index of the first such region within the _region_data array. - // if (_generation->generation_mode() != OLD) these two variables are - // not used. - uint _old_coalesce_and_fill_candidates; - uint _first_coalesce_and_fill_candidate; + // This is the last index in the array of old regions which were active at + // the end of old final mark. + uint _last_old_region; // This can be the 'static' or 'adaptive' heuristic. ShenandoahHeuristics* _trigger_heuristic; - // Flag is set when promotion failure is detected (by gc thread), cleared when old generation collection begins (by control thread) + // Flag is set when promotion failure is detected (by gc thread), and cleared when + // old generation collection begins (by control thread). volatile bool _promotion_failed; - // Prepare for evacuation of old-gen regions by capturing the mark results of a recently completed concurrent mark pass. - void prepare_for_old_collections(); + // Keep a pointer to our generation that we can use without down casting a protected member from the base class. + ShenandoahOldGeneration* _old_generation; protected: virtual void choose_collection_set_from_regiondata(ShenandoahCollectionSet* set, RegionData* data, size_t data_size, size_t free) override; public: - ShenandoahOldHeuristics(ShenandoahGeneration* generation, ShenandoahHeuristics* trigger_heuristic); + ShenandoahOldHeuristics(ShenandoahOldGeneration* generation, ShenandoahHeuristics* trigger_heuristic); virtual void choose_collection_set(ShenandoahCollectionSet* collection_set, ShenandoahOldHeuristics* old_heuristics) override; + // Prepare for evacuation of old-gen regions by capturing the mark results of a recently completed concurrent mark pass. + void prepare_for_old_collections(); + // Return true iff the collection set is primed with at least one old-gen region. bool prime_collection_set(ShenandoahCollectionSet* set); - // Having coalesced and filled all old-gen heap regions that are not part of the old-gen collection set, begin - // evacuating the collection set. - void start_old_evacuations(); - // How many old-collection candidates have not yet been processed? uint unprocessed_old_collection_candidates(); // How many old or hidden collection candidates have not yet been processed? - uint unprocessed_old_or_hidden_collection_candidates(); + uint last_old_collection_candidate_index(); // Return the next old-collection candidate in order of decreasing amounts of garbage. (We process most-garbage regions // first.) This does not consume the candidate. If the candidate is selected for inclusion in a collection set, then @@ -104,13 +114,13 @@ class ShenandoahOldHeuristics : public ShenandoahHeuristics { // How many old-collection regions were identified at the end of the most recent old-gen mark to require their // unmarked objects to be coalesced and filled? - uint old_coalesce_and_fill_candidates(); + uint last_old_region_index() const; // Fill in buffer with all of the old-collection regions that were identified at the end of the most recent old-gen // mark to require their unmarked objects to be coalesced and filled. The buffer array must have at least - // old_coalesce_and_fill_candidates() entries, or memory may be corrupted when this function overwrites the + // last_old_region_index() entries, or memory may be corrupted when this function overwrites the // end of the array. - void get_coalesce_and_fill_candidates(ShenandoahHeapRegion** buffer); + unsigned int get_coalesce_and_fill_candidates(ShenandoahHeapRegion** buffer); // If a GLOBAL gc occurs, it will collect the entire heap which invalidates any collection candidates being // held by this heuristic for supplying mixed collections. @@ -151,6 +161,8 @@ class ShenandoahOldHeuristics : public ShenandoahHeuristics { virtual bool is_experimental() override; + private: + void slide_pinned_regions_to_front(); }; #endif // SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHOLDHEURISTICS_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index e7a2ec01bd120..b836a551eb7da 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -258,7 +258,7 @@ void ShenandoahConcurrentGC::vmop_entry_init_mark() { ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::init_mark_gross); heap->try_inject_alloc_failure(); - VM_ShenandoahInitMark op(this, _do_old_gc_bootstrap); + VM_ShenandoahInitMark op(this); VMThread::execute(&op); // jump to entry_init_mark() under safepoint } @@ -567,7 +567,7 @@ void ShenandoahConcurrentGC::op_reset() { if (ShenandoahPacing) { heap->pacer()->setup_for_reset(); } - _generation->prepare_gc(_do_old_gc_bootstrap); + _generation->prepare_gc(); } class ShenandoahInitMarkUpdateRegionStateClosure : public ShenandoahHeapRegionClosure { @@ -642,10 +642,13 @@ void ShenandoahConcurrentGC::op_init_mark() { if (_do_old_gc_bootstrap) { // Update region state for both young and old regions + // TODO: We should be able to pull this out of the safepoint for the bootstrap + // cycle. The top of an old region will only move when a GC cycle evacuates + // objects into it. When we start an old cycle, we know that nothing can touch + // the top of old regions. ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states); ShenandoahInitMarkUpdateRegionStateClosure cl; heap->parallel_heap_region_iterate(&cl); - heap->old_generation()->parallel_heap_region_iterate(&cl); } else { // Update region state for only young regions ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_update_region_states); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index b8e43ea742509..e82859ca948dd 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -32,6 +32,7 @@ #include "gc/shenandoah/shenandoahFullGC.hpp" #include "gc/shenandoah/shenandoahGeneration.hpp" #include "gc/shenandoah/shenandoahYoungGeneration.hpp" +#include "gc/shenandoah/shenandoahOldGeneration.hpp" #include "gc/shenandoah/shenandoahPhaseTimings.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahMark.inline.hpp" @@ -212,7 +213,7 @@ void ShenandoahControlThread::run_service() { // Don't start a new old marking if there is one already in progress. if (generation == OLD && heap->is_concurrent_old_mark_in_progress()) { - set_gc_mode(marking_old); + set_gc_mode(servicing_old); } if (generation == GLOBAL) { @@ -226,16 +227,16 @@ void ShenandoahControlThread::run_service() { // blocking 'request_gc' method, but there it loops and resets the // '_requested_gc_cause' until a full cycle is completed. _requested_gc_cause = GCCause::_no_gc; - } else if (heap->is_concurrent_old_mark_in_progress() || heap->is_concurrent_prep_for_mixed_evacuation_in_progress()) { + } else if (heap->is_concurrent_old_mark_in_progress() || heap->is_prepare_for_old_mark_in_progress()) { // Nobody asked us to do anything, but we have an old-generation mark or old-generation preparation for // mixed evacuation in progress, so resume working on that. log_info(gc)("Resume old gc: marking=%s, preparing=%s", BOOL_TO_STR(heap->is_concurrent_old_mark_in_progress()), - BOOL_TO_STR(heap->is_concurrent_prep_for_mixed_evacuation_in_progress())); + BOOL_TO_STR(heap->is_prepare_for_old_mark_in_progress())); cause = GCCause::_shenandoah_concurrent_gc; generation = OLD; - set_gc_mode(marking_old); + set_gc_mode(servicing_old); } } @@ -288,9 +289,9 @@ void ShenandoahControlThread::run_service() { service_stw_full_cycle(cause); break; } - case marking_old: { + case servicing_old: { assert(generation == OLD, "Expected old generation here"); - resume_concurrent_old_cycle(heap->old_generation(), cause); + service_concurrent_old_cycle(heap, cause); break; } default: { @@ -432,9 +433,9 @@ void ShenandoahControlThread::process_phase_timings(const ShenandoahHeap* heap) // | | | | | | // | v v v v | // | Resume Old <----------+ Young +--> Young Degen | -// | + + + | -// v | | | | -// Global <-+ | | | +// | + + ^ + + | +// v | | | | | | +// Global <-+ | +----------------------------+ | | // + | | | // | v v | // +---> Global Degen +--------------------> Full <----+ @@ -472,83 +473,88 @@ void ShenandoahControlThread::service_concurrent_normal_cycle( } void ShenandoahControlThread::service_concurrent_old_cycle(const ShenandoahHeap* heap, GCCause::Cause &cause) { - // Configure the young generation's concurrent mark to put objects in - // old regions into the concurrent mark queues associated with the old - // generation. The young cycle will run as normal except that rather than - // ignore old references it will mark and enqueue them in the old concurrent - // mark but it will not traverse them. - ShenandoahGeneration* old_generation = heap->old_generation(); + + ShenandoahOldGeneration* old_generation = heap->old_generation(); ShenandoahYoungGeneration* young_generation = heap->young_generation(); - assert(!heap->is_concurrent_old_mark_in_progress(), "Old already in progress."); - assert(old_generation->task_queues()->is_empty(), "Old mark queues should be empty."); + GCIdMark gc_id_mark; + TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); - young_generation->set_old_gen_task_queues(old_generation->task_queues()); - young_generation->set_mark_incomplete(); - old_generation->set_mark_incomplete(); - service_concurrent_cycle(young_generation, cause, true); + switch (old_generation->state()) { + case ShenandoahOldGeneration::IDLE: { + assert(!heap->is_concurrent_old_mark_in_progress(), "Old already in progress."); + assert(old_generation->task_queues()->is_empty(), "Old mark queues should be empty."); + } + case ShenandoahOldGeneration::FILLING: { + _allow_old_preemption.set(); + ShenandoahGCSession session(cause, old_generation); + old_generation->prepare_gc(); + _allow_old_preemption.unset(); + + if (heap->is_prepare_for_old_mark_in_progress()) { + assert(old_generation->state() == ShenandoahOldGeneration::FILLING, "Prepare for mark should be in progress."); + return; + } - process_phase_timings(heap); + assert(old_generation->state() == ShenandoahOldGeneration::BOOTSTRAPPING, "Finished with filling, should be bootstrapping."); + } + case ShenandoahOldGeneration::BOOTSTRAPPING: { + // Configure the young generation's concurrent mark to put objects in + // old regions into the concurrent mark queues associated with the old + // generation. The young cycle will run as normal except that rather than + // ignore old references it will mark and enqueue them in the old concurrent + // task queues but it will not traverse them. + young_generation->set_old_gen_task_queues(old_generation->task_queues()); + ShenandoahGCSession session(cause, young_generation); + service_concurrent_cycle(heap,young_generation, cause, true); + process_phase_timings(heap); + if (heap->cancelled_gc()) { + // Young generation bootstrap cycle has failed. Concurrent mark for old generation + // is going to resume after degenerated bootstrap cycle completes. + log_info(gc)("Bootstrap cycle for old generation was cancelled."); + return; + } - if (heap->cancelled_gc()) { - // Young generation bootstrap cycle has failed. Concurrent mark for old generation - // is not going to resume after degenerated young cycle completes. - log_info(gc)("Bootstrap cycle for old generation was cancelled."); - } else { - // Reset the degenerated point. Normally this would happen at the top - // of the control loop, but here we have just completed a young cycle - // which has bootstrapped the old concurrent marking. - _degen_point = ShenandoahGC::_degenerated_outside_cycle; - - // From here we will 'resume' the old concurrent mark. This will skip reset - // and init mark for the concurrent mark. All of that work will have been - // done by the bootstrapping young cycle. In order to simplify the debugging - // effort, the old cycle will ONLY complete the mark phase. No actual - // collection of the old generation is happening here. - set_gc_mode(marking_old); - resume_concurrent_old_cycle(old_generation, cause); - } -} + // Reset the degenerated point. Normally this would happen at the top + // of the control loop, but here we have just completed a young cycle + // which has bootstrapped the old concurrent marking. + _degen_point = ShenandoahGC::_degenerated_outside_cycle; -bool ShenandoahControlThread::check_soft_max_changed() const { - ShenandoahHeap* heap = ShenandoahHeap::heap(); - size_t new_soft_max = Atomic::load(&SoftMaxHeapSize); - size_t old_soft_max = heap->soft_max_capacity(); - if (new_soft_max != old_soft_max) { - new_soft_max = MAX2(heap->min_capacity(), new_soft_max); - new_soft_max = MIN2(heap->max_capacity(), new_soft_max); - if (new_soft_max != old_soft_max) { - log_info(gc)("Soft Max Heap Size: " SIZE_FORMAT "%s -> " SIZE_FORMAT "%s", - byte_size_in_proper_unit(old_soft_max), proper_unit_for_byte_size(old_soft_max), - byte_size_in_proper_unit(new_soft_max), proper_unit_for_byte_size(new_soft_max) - ); - heap->set_soft_max_capacity(new_soft_max); - return true; + // From here we will 'resume' the old concurrent mark. This will skip reset + // and init mark for the concurrent mark. All of that work will have been + // done by the bootstrapping young cycle. In order to simplify the debugging + // effort, the old cycle will ONLY complete the mark phase. No actual + // collection of the old generation is happening here. + set_gc_mode(servicing_old); + old_generation->transition_to(ShenandoahOldGeneration::MARKING); + } + case ShenandoahOldGeneration::MARKING: { + ShenandoahGCSession session(cause, old_generation); + bool marking_complete = resume_concurrent_old_cycle(old_generation, cause); + if (marking_complete) { + assert(old_generation->state() != ShenandoahOldGeneration::MARKING, "Should not still be marking."); + } + break; } + default: + log_error(gc)("Unexpected state for old GC: %d", old_generation->state()); + ShouldNotReachHere(); } - return false; } -void ShenandoahControlThread::resume_concurrent_old_cycle(ShenandoahGeneration* generation, GCCause::Cause cause) { +bool ShenandoahControlThread::resume_concurrent_old_cycle(ShenandoahGeneration* generation, GCCause::Cause cause) { - assert(ShenandoahHeap::heap()->is_concurrent_old_mark_in_progress() || - ShenandoahHeap::heap()->is_concurrent_prep_for_mixed_evacuation_in_progress(), - "Old mark or mixed-evac prep should be in progress"); + assert(ShenandoahHeap::heap()->is_concurrent_old_mark_in_progress(), "Old mark should be in progress"); log_debug(gc)("Resuming old generation with " UINT32_FORMAT " marking tasks queued.", generation->task_queues()->tasks()); ShenandoahHeap* heap = ShenandoahHeap::heap(); - GCIdMark gc_id_mark; - ShenandoahGCSession session(cause, generation); - - TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); // We can only tolerate being cancelled during concurrent marking or during preparation for mixed // evacuation. This flag here (passed by reference) is used to control precisely where the regulator // is allowed to cancel a GC. ShenandoahOldGC gc(generation, _allow_old_preemption); if (gc.collect(cause)) { - generation->heuristics()->record_success_concurrent(false); - heap->shenandoah_policy()->record_success_old(); + generation->record_success_concurrent(false); } if (heap->cancelled_gc()) { @@ -565,7 +571,28 @@ void ShenandoahControlThread::resume_concurrent_old_cycle(ShenandoahGeneration* if (_requested_gc_cause == GCCause::_shenandoah_concurrent_gc) { heap->shenandoah_policy()->record_interrupted_old(); } + return false; + } + return true; +} + +bool ShenandoahControlThread::check_soft_max_changed() const { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + size_t new_soft_max = Atomic::load(&SoftMaxHeapSize); + size_t old_soft_max = heap->soft_max_capacity(); + if (new_soft_max != old_soft_max) { + new_soft_max = MAX2(heap->min_capacity(), new_soft_max); + new_soft_max = MIN2(heap->max_capacity(), new_soft_max); + if (new_soft_max != old_soft_max) { + log_info(gc)("Soft Max Heap Size: " SIZE_FORMAT "%s -> " SIZE_FORMAT "%s", + byte_size_in_proper_unit(old_soft_max), proper_unit_for_byte_size(old_soft_max), + byte_size_in_proper_unit(new_soft_max), proper_unit_for_byte_size(new_soft_max) + ); + heap->set_soft_max_capacity(new_soft_max); + return true; + } } + return false; } void ShenandoahControlThread::service_concurrent_cycle(ShenandoahGeneration* generation, GCCause::Cause cause, bool do_old_gc_bootstrap) { @@ -604,19 +631,22 @@ void ShenandoahControlThread::service_concurrent_cycle(ShenandoahGeneration* gen // v | // Full GC --------------------------/ // - ShenandoahHeap* heap = ShenandoahHeap::heap(); if (check_cancellation_or_degen(ShenandoahGC::_degenerated_outside_cycle)) return; + ShenandoahHeap* heap = ShenandoahHeap::heap(); GCIdMark gc_id_mark; ShenandoahGCSession session(cause, generation); - TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); + service_concurrent_cycle(heap, generation, cause, do_old_gc_bootstrap); +} + +void ShenandoahControlThread::service_concurrent_cycle(const ShenandoahHeap* heap, ShenandoahGeneration* generation, + GCCause::Cause &cause, bool do_old_gc_bootstrap) { ShenandoahConcurrentGC gc(generation, do_old_gc_bootstrap); if (gc.collect(cause)) { // Cycle is complete - generation->heuristics()->record_success_concurrent(gc.abbreviated()); - heap->shenandoah_policy()->record_success_concurrent(); + generation->record_success_concurrent(gc.abbreviated()); } else { assert(heap->cancelled_gc(), "Must have been cancelled"); check_cancellation_or_degen(gc.degen_point()); @@ -694,6 +724,12 @@ bool ShenandoahControlThread::service_stw_degenerated_cycle(GCCause::Cause cause if (_degen_generation->generation_mode() == GLOBAL) { assert(heap->old_generation()->task_queues()->is_empty(), "Unexpected old generation marking tasks"); assert(heap->global_generation()->task_queues()->is_empty(), "Unexpected global generation marking tasks"); + } else { + assert(_degen_generation->generation_mode() == YOUNG, "Expected degenerated young cycle, if not global."); + ShenandoahOldGeneration* old_generation = (ShenandoahOldGeneration*) heap->old_generation(); + if (old_generation->state() == ShenandoahOldGeneration::BOOTSTRAPPING && !gc.upgraded_to_full()) { + old_generation->transition_to(ShenandoahOldGeneration::MARKING); + } } _degen_generation->heuristics()->record_success_degenerated(); @@ -947,7 +983,7 @@ const char* ShenandoahControlThread::gc_mode_name(ShenandoahControlThread::GCMod case concurrent_normal: return "normal"; case stw_degenerated: return "degenerated"; case stw_full: return "full"; - case marking_old: return "old mark"; + case servicing_old: return "old"; default: return "unknown"; } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp index e0e22288d39a6..73209e991e378 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp @@ -73,7 +73,7 @@ class ShenandoahControlThread: public ConcurrentGCThread { concurrent_normal, stw_degenerated, stw_full, - marking_old + servicing_old } GCMode; void run_service(); @@ -102,8 +102,11 @@ class ShenandoahControlThread: public ConcurrentGCThread { volatile GCMode _mode; shenandoah_padding(3); + // Returns true if the cycle has been cancelled or degenerated. bool check_cancellation_or_degen(ShenandoahGC::ShenandoahDegenPoint point); - void resume_concurrent_old_cycle(ShenandoahGeneration* generation, GCCause::Cause cause); + + // Returns true if the old generation marking completed (i.e., final mark executed for old generation). + bool resume_concurrent_old_cycle(ShenandoahGeneration* generation, GCCause::Cause cause); void service_concurrent_cycle(ShenandoahGeneration* generation, GCCause::Cause cause, bool reset_old_bitmap_specially); void service_stw_full_cycle(GCCause::Cause cause); @@ -112,8 +115,11 @@ class ShenandoahControlThread: public ConcurrentGCThread { bool service_stw_degenerated_cycle(GCCause::Cause cause, ShenandoahGC::ShenandoahDegenPoint point); void service_uncommit(double shrink_before, size_t shrink_until); + // Return true if setting the flag which indicates allocation failure succeeds. bool try_set_alloc_failure_gc(); + // Notify threads waiting for GC to complete. void notify_alloc_failure_waiters(); + // True if allocation failure flag has been set. bool is_alloc_failure_gc(); void reset_gc_id(); @@ -128,8 +134,10 @@ class ShenandoahControlThread: public ConcurrentGCThread { bool is_explicit_gc(GCCause::Cause cause) const; bool is_implicit_gc(GCCause::Cause cause) const; + // Returns true if the old generation marking was interrupted to allow a young cycle. bool preempt_old_marking(GenerationMode generation); + // Returns true if the soft maximum heap has been changed using management APIs. bool check_soft_max_changed() const; void process_phase_timings(const ShenandoahHeap* heap); @@ -148,6 +156,7 @@ class ShenandoahControlThread: public ConcurrentGCThread { void handle_alloc_failure_evac(size_t words); void request_gc(GCCause::Cause cause); + // Return true if the request to start a concurrent GC for the given generation succeeded. bool request_concurrent_gc(GenerationMode generation); void handle_counters_update(); @@ -183,6 +192,9 @@ class ShenandoahControlThread: public ConcurrentGCThread { private: static const char* gc_mode_name(GCMode mode); void notify_control_thread(); + + void service_concurrent_cycle(const ShenandoahHeap* heap, ShenandoahGeneration* generation, GCCause::Cause &cause, + bool do_old_gc_bootstrap); }; #endif // SHARE_GC_SHENANDOAH_SHENANDOAHCONTROLTHREAD_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp index e9af9f585a3b5..c949f4cb3d297 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp @@ -33,6 +33,7 @@ #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahMetrics.hpp" #include "gc/shenandoah/shenandoahMonitoringSupport.hpp" +#include "gc/shenandoah/shenandoahOldGeneration.hpp" #include "gc/shenandoah/shenandoahOopClosures.inline.hpp" #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp" #include "gc/shenandoah/shenandoahSTWMark.hpp" @@ -319,7 +320,7 @@ void ShenandoahDegenGC::op_degenerated() { } void ShenandoahDegenGC::op_reset() { - _generation->prepare_gc(false); + _generation->prepare_gc(); } void ShenandoahDegenGC::op_mark() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index b20b323c91f95..6fd770697a11e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -29,6 +29,7 @@ #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahHeapRegionSet.hpp" #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp" +#include "gc/shenandoah/shenandoahOldGeneration.hpp" #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp" #include "gc/shenandoah/shenandoahYoungGeneration.hpp" #include "logging/logStream.hpp" diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index 9beb5c8c9916d..ee1bcd0964416 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -44,6 +44,7 @@ #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp" #include "gc/shenandoah/shenandoahMetrics.hpp" +#include "gc/shenandoah/shenandoahOldGeneration.hpp" #include "gc/shenandoah/shenandoahOopClosures.inline.hpp" #include "gc/shenandoah/shenandoahReferenceProcessor.hpp" #include "gc/shenandoah/shenandoahRootProcessor.inline.hpp" diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index 2d3f27f70f1a6..3502288b06144 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -23,12 +23,13 @@ */ #include "precompiled.hpp" - +#include "gc/shenandoah/shenandoahCollectorPolicy.hpp" #include "gc/shenandoah/shenandoahFreeSet.hpp" #include "gc/shenandoah/shenandoahGeneration.hpp" #include "gc/shenandoah/shenandoahHeap.hpp" #include "gc/shenandoah/shenandoahMarkClosures.hpp" #include "gc/shenandoah/shenandoahMonitoringSupport.hpp" +#include "gc/shenandoah/shenandoahOldGeneration.hpp" #include "gc/shenandoah/shenandoahReferenceProcessor.hpp" #include "gc/shenandoah/shenandoahTaskqueue.inline.hpp" #include "gc/shenandoah/shenandoahUtils.hpp" @@ -61,7 +62,7 @@ class ShenandoahResetBitmapTask : public ShenandoahHeapRegionClosure { ShenandoahMarkingContext* const _ctx; public: ShenandoahResetBitmapTask() : - _heap(ShenandoahHeap::heap()), + _heap(ShenandoahHeap::heap()), _ctx(_heap->marking_context()) {} void heap_region_do(ShenandoahHeapRegion* region) { @@ -204,21 +205,12 @@ void ShenandoahGeneration::merge_write_table() { heap->old_generation()->parallel_heap_region_iterate(&task); } -void ShenandoahGeneration::prepare_gc(bool do_old_gc_bootstrap) { +void ShenandoahGeneration::prepare_gc() { // Reset mark bitmap for this generation (typically young) reset_mark_bitmap(); - if (do_old_gc_bootstrap) { - // Reset mark bitmap for old regions also. Note that do_old_gc_bootstrap is only true if this generation is YOUNG. - ShenandoahHeap::heap()->old_generation()->reset_mark_bitmap(); - } - // Capture Top At Mark Start for this generation (typically young) ShenandoahResetUpdateRegionStateClosure cl; parallel_heap_region_iterate(&cl); - if (do_old_gc_bootstrap) { - // Capture top at mark start for both old-gen regions also. Note that do_old_gc_bootstrap is only true if generation is YOUNG. - ShenandoahHeap::heap()->old_generation()->parallel_heap_region_iterate(&cl); - } } void ShenandoahGeneration::compute_evacuation_budgets(ShenandoahHeap* heap, bool* preselected_regions, @@ -420,7 +412,7 @@ void ShenandoahGeneration::adjust_evacuation_budgets(ShenandoahHeap* heap, Shena size_t minimum_evacuation_reserve, size_t consumed_by_advance_promotion) { if (heap->mode()->is_generational()) { size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); - ShenandoahGeneration* old_generation = heap->old_generation(); + ShenandoahOldGeneration* old_generation = heap->old_generation(); ShenandoahYoungGeneration* young_generation = heap->young_generation(); size_t old_evacuated = collection_set->get_old_bytes_reserved_for_evacuation(); size_t old_evacuated_committed = (size_t) (ShenandoahEvacWaste * old_evacuated); @@ -649,17 +641,20 @@ void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) { assert(generation_mode() != OLD, "Only YOUNG and GLOBAL GC perform evacuations"); { ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_update_region_states : - ShenandoahPhaseTimings::degen_gc_final_update_region_states); + ShenandoahPhaseTimings::degen_gc_final_update_region_states); ShenandoahFinalMarkUpdateRegionStateClosure cl(complete_marking_context()); - parallel_heap_region_iterate(&cl); - heap->assert_pinned_region_status(); if (generation_mode() == YOUNG) { - // Also capture update_watermark for old-gen regions. - ShenandoahCaptureUpdateWaterMarkForOld old_cl(complete_marking_context()); + // We always need to update the watermark for old regions. If there + // are mixed collections pending, we also need to synchronize the + // pinned status for old regions. Since we are already visiting every + // old region here, go ahead and sync the pin status too. + ShenandoahFinalMarkUpdateRegionStateClosure old_cl(nullptr); heap->old_generation()->parallel_heap_region_iterate(&old_cl); } + + heap->assert_pinned_region_status(); } { @@ -856,3 +851,13 @@ size_t ShenandoahGeneration::adjusted_available() const { size_t capacity = _adjusted_capacity; return in_use > capacity ? 0 : capacity - in_use; } + +void ShenandoahGeneration::record_success_concurrent(bool abbreviated) { + heuristics()->record_success_concurrent(abbreviated); + ShenandoahHeap::heap()->shenandoah_policy()->record_success_concurrent(); +} + +void ShenandoahGeneration::record_success_degenerated() { + heuristics()->record_success_degenerated(); + ShenandoahHeap::heap()->shenandoah_policy()->record_success_degenerated(); +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index 9b69bcfdf6d7b..e2068883852f3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -120,10 +120,10 @@ class ShenandoahGeneration : public CHeapObj { // Update the read cards with the state of the write table (write table is not cleared). void merge_write_table(); - // Used by concurrent and degenerated GC to reset regions. - void prepare_gc(bool do_old_gc_bootstrap); + // Called before init mark, expected to prepare regions for marking. + virtual void prepare_gc(); - // Return true iff prepared collection set includes at least one old-gen HeapRegion. + // Called during final mark, chooses collection set, rebuilds free set. virtual void prepare_regions_and_collection_set(bool concurrent); // Cancel marking (used by Full collect and when cancelling cycle). @@ -171,6 +171,9 @@ class ShenandoahGeneration : public CHeapObj { virtual bool is_concurrent_mark_in_progress() = 0; void confirm_heuristics_mode(); + + virtual void record_success_concurrent(bool abbreviated); + virtual void record_success_degenerated(); }; #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHGENERATION_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 8d2c7cba07ace..b7994a5bdce54 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -498,7 +498,7 @@ void ShenandoahHeap::initialize_heuristics() { ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) : CollectedHeap(), _gc_generation(NULL), - _prep_for_mixed_evac_in_progress(false), + _prepare_for_old_mark(false), _initial_size(0), _used(0), _committed(0), @@ -642,6 +642,12 @@ bool ShenandoahHeap::doing_mixed_evacuations() { return old_heuristics()->unprocessed_old_collection_candidates() > 0; } +bool ShenandoahHeap::is_old_bitmap_stable() const { + ShenandoahOldGeneration::State state = _old_generation->state(); + return state != ShenandoahOldGeneration::MARKING + && state != ShenandoahOldGeneration::BOOTSTRAPPING; +} + bool ShenandoahHeap::is_gc_generation_young() const { return _gc_generation != NULL && _gc_generation->generation_mode() == YOUNG; } @@ -1030,18 +1036,20 @@ void ShenandoahHeap::cancel_old_gc() { // Stop marking old_generation()->cancel_marking(); // Stop coalescing undead objects - set_concurrent_prep_for_mixed_evacuation_in_progress(false); + set_prepare_for_old_mark_in_progress(false); // Stop tracking old regions old_heuristics()->abandon_collection_candidates(); // Remove old generation access to young generation mark queues young_generation()->set_old_gen_task_queues(nullptr); + // Transition to IDLE now. + _old_generation->transition_to(ShenandoahOldGeneration::IDLE); } bool ShenandoahHeap::is_old_gc_active() { return is_concurrent_old_mark_in_progress() - || is_concurrent_prep_for_mixed_evacuation_in_progress() - || old_heuristics()->unprocessed_old_or_hidden_collection_candidates() > 0 - || young_generation()->old_gen_task_queues() != nullptr; + || is_prepare_for_old_mark_in_progress() + || old_heuristics()->unprocessed_old_collection_candidates() > 0 + || young_generation()->old_gen_task_queues() != nullptr; } void ShenandoahHeap::coalesce_and_fill_old_regions() { @@ -2123,14 +2131,10 @@ void ShenandoahHeap::set_concurrent_old_mark_in_progress(bool in_progress) { manage_satb_barrier(in_progress); } -void ShenandoahHeap::set_concurrent_prep_for_mixed_evacuation_in_progress(bool in_progress) { +void ShenandoahHeap::set_prepare_for_old_mark_in_progress(bool in_progress) { // Unlike other set-gc-state functions, this may happen outside safepoint. // Is only set and queried by control thread, so no coherence issues. - _prep_for_mixed_evac_in_progress = in_progress; -} - -bool ShenandoahHeap::is_concurrent_prep_for_mixed_evacuation_in_progress() { - return _prep_for_mixed_evac_in_progress; + _prepare_for_old_mark = in_progress; } void ShenandoahHeap::set_aging_cycle(bool in_progress) { @@ -2981,7 +2985,7 @@ bool ShenandoahHeap::requires_barriers(stackChunkOop obj) const { } void ShenandoahHeap::transfer_old_pointers_from_satb() { - ((ShenandoahOldGeneration*) _old_generation)->transfer_pointers_from_satb(); + _old_generation->transfer_pointers_from_satb(); } template<> @@ -3020,8 +3024,7 @@ void ShenandoahHeap::verify_rem_set_at_mark() { log_debug(gc)("Verifying remembered set at %s mark", doing_mixed_evacuations()? "mixed": "young"); - if (doing_mixed_evacuations() || - is_concurrent_prep_for_mixed_evacuation_in_progress() || active_generation()->generation_mode() == GLOBAL) { + if (is_old_bitmap_stable() || active_generation()->generation_mode() == GLOBAL) { ctx = complete_marking_context(); } else { ctx = nullptr; @@ -3155,8 +3158,7 @@ void ShenandoahHeap::verify_rem_set_at_update_ref() { ShenandoahRegionIterator iterator; ShenandoahMarkingContext* ctx; - if (doing_mixed_evacuations() || - is_concurrent_prep_for_mixed_evacuation_in_progress() || active_generation()->generation_mode() == GLOBAL) { + if (is_old_bitmap_stable() || active_generation()->generation_mode() == GLOBAL) { ctx = complete_marking_context(); } else { ctx = nullptr; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index 3a3196030627e..2f6fb2f550d9f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -52,6 +52,7 @@ class ShenandoahGCSession; class ShenandoahGCStateResetter; class ShenandoahGeneration; class ShenandoahYoungGeneration; +class ShenandoahOldGeneration; class ShenandoahHeuristics; class ShenandoahOldHeuristics; class ShenandoahMarkingContext; @@ -151,7 +152,8 @@ class ShenandoahHeap : public CollectedHeap { ShenandoahHeapLock _lock; ShenandoahGeneration* _gc_generation; - bool _prep_for_mixed_evac_in_progress; // true iff we are concurrently coalescing and filling old-gen HeapRegions + // true iff we are concurrently coalescing and filling old-gen HeapRegions + bool _prepare_for_old_mark; public: ShenandoahHeapLock* lock() { @@ -170,7 +172,7 @@ class ShenandoahHeap : public CollectedHeap { ShenandoahOldHeuristics* old_heuristics(); bool doing_mixed_evacuations(); - + bool is_old_bitmap_stable() const; bool is_gc_generation_young() const; // ---------- Initialization, termination, identification, printing routines @@ -397,7 +399,7 @@ class ShenandoahHeap : public CollectedHeap { void set_has_forwarded_objects(bool cond); void set_concurrent_strong_root_in_progress(bool cond); void set_concurrent_weak_root_in_progress(bool cond); - void set_concurrent_prep_for_mixed_evacuation_in_progress(bool cond); + void set_prepare_for_old_mark_in_progress(bool cond); void set_aging_cycle(bool cond); inline bool is_stable() const; @@ -415,7 +417,7 @@ class ShenandoahHeap : public CollectedHeap { inline bool is_stw_gc_in_progress() const; inline bool is_concurrent_strong_root_in_progress() const; inline bool is_concurrent_weak_root_in_progress() const; - bool is_concurrent_prep_for_mixed_evacuation_in_progress(); + inline bool is_prepare_for_old_mark_in_progress() const; inline bool is_aging_cycle() const; inline bool upgraded_to_full() { return _upgraded_to_full; } inline void start_conc_gc() { _upgraded_to_full = false; } @@ -519,7 +521,7 @@ class ShenandoahHeap : public CollectedHeap { private: ShenandoahYoungGeneration* _young_generation; ShenandoahGeneration* _global_generation; - ShenandoahGeneration* _old_generation; + ShenandoahOldGeneration* _old_generation; ShenandoahControlThread* _control_thread; ShenandoahRegulatorThread* _regulator_thread; @@ -537,7 +539,7 @@ class ShenandoahHeap : public CollectedHeap { public: ShenandoahYoungGeneration* young_generation() const { return _young_generation; } ShenandoahGeneration* global_generation() const { return _global_generation; } - ShenandoahGeneration* old_generation() const { return _old_generation; } + ShenandoahOldGeneration* old_generation() const { return _old_generation; } ShenandoahGeneration* generation_for(ShenandoahRegionAffiliation affiliation) const; ShenandoahCollectorPolicy* shenandoah_policy() const { return _shenandoah_policy; } @@ -565,7 +567,7 @@ class ShenandoahHeap : public CollectedHeap { // For exporting to SA int _log_min_obj_alignment_in_bytes; public: - ShenandoahMonitoringSupport* monitoring_support() { return _monitoring_support; } + ShenandoahMonitoringSupport* monitoring_support() const { return _monitoring_support; } GCMemoryManager* cycle_memory_manager() { return &_cycle_memory_manager; } GCMemoryManager* stw_memory_manager() { return &_stw_memory_manager; } SoftRefPolicy* soft_ref_policy() { return &_soft_ref_policy; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp index 01bf3241bc90e..076dd6c22653b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp @@ -671,6 +671,10 @@ inline bool ShenandoahHeap::is_aging_cycle() const { return _is_aging_cycle.is_set(); } +inline bool ShenandoahHeap::is_prepare_for_old_mark_in_progress() const { + return _prepare_for_old_mark; +} + inline size_t ShenandoahHeap::set_promoted_reserve(size_t new_val) { size_t orig = _promoted_reserve; _promoted_reserve = new_val; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index 19b0317a89157..8cce1a6ffb938 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -30,6 +30,7 @@ #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahHeapRegion.hpp" #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp" +#include "gc/shenandoah/shenandoahOldGeneration.hpp" #include "gc/shenandoah/shenandoahGeneration.hpp" #include "gc/shenandoah/shenandoahYoungGeneration.hpp" #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp" diff --git a/src/hotspot/share/gc/shenandoah/shenandoahInitLogger.cpp b/src/hotspot/share/gc/shenandoah/shenandoahInitLogger.cpp index 472883e33e167..b7acc53307768 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahInitLogger.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahInitLogger.cpp @@ -26,7 +26,7 @@ #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahHeapRegion.hpp" #include "gc/shenandoah/shenandoahInitLogger.hpp" -#include "gc/shenandoah/shenandoahGeneration.hpp" +#include "gc/shenandoah/shenandoahOldGeneration.hpp" #include "gc/shenandoah/shenandoahYoungGeneration.hpp" #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" #include "gc/shenandoah/mode/shenandoahMode.hpp" diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.cpp index db9f309e6f07c..ba0dadf6fc148 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.cpp @@ -37,12 +37,19 @@ ShenandoahFinalMarkUpdateRegionStateClosure::ShenandoahFinalMarkUpdateRegionStat void ShenandoahFinalMarkUpdateRegionStateClosure::heap_region_do(ShenandoahHeapRegion* r) { if (r->is_active()) { - // All allocations past TAMS are implicitly live, adjust the region data. - // Bitmaps/TAMS are swapped at this point, so we need to poll complete bitmap. - HeapWord *tams = _ctx->top_at_mark_start(r); - HeapWord *top = r->top(); - if (top > tams) { - r->increase_live_data_alloc_words(pointer_delta(top, tams)); + if (_ctx != nullptr) { + // _ctx may be null when this closure is used to sync only the pin status + // update the watermark of old regions. For old regions we cannot reset + // the TAMS because we rely on that to keep promoted objects alive after + // old marking is complete. + + // All allocations past TAMS are implicitly live, adjust the region data. + // Bitmaps/TAMS are swapped at this point, so we need to poll complete bitmap. + HeapWord *tams = _ctx->top_at_mark_start(r); + HeapWord *top = r->top(); + if (top > tams) { + r->increase_live_data_alloc_words(pointer_delta(top, tams)); + } } // We are about to select the collection set, make sure it knows about @@ -65,15 +72,7 @@ void ShenandoahFinalMarkUpdateRegionStateClosure::heap_region_do(ShenandoahHeapR r->set_update_watermark_at_safepoint(r->top()); } else { assert(!r->has_live(), "Region " SIZE_FORMAT " should have no live data", r->index()); - assert(_ctx->top_at_mark_start(r) == r->top(), - "Region " SIZE_FORMAT " should have correct TAMS", r->index()); + assert(_ctx == nullptr || _ctx->top_at_mark_start(r) == r->top(), + "Region " SIZE_FORMAT " should have correct TAMS", r->index()); } } - -ShenandoahCaptureUpdateWaterMarkForOld::ShenandoahCaptureUpdateWaterMarkForOld(ShenandoahMarkingContext* ctx) : - _ctx(ctx), _lock(ShenandoahHeap::heap()->lock()) {} - -void ShenandoahCaptureUpdateWaterMarkForOld::heap_region_do(ShenandoahHeapRegion* r) { - // Remember limit for updating refs. It's guaranteed that we get no from-space-refs written from here on. - r->set_update_watermark_at_safepoint(r->top()); -} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.hpp index 6aa013fb3dc0c..304541cb7e39c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.hpp @@ -35,19 +35,7 @@ class ShenandoahFinalMarkUpdateRegionStateClosure : public ShenandoahHeapRegionC ShenandoahMarkingContext* const _ctx; ShenandoahHeapLock* const _lock; public: - ShenandoahFinalMarkUpdateRegionStateClosure(ShenandoahMarkingContext* ctx); - - void heap_region_do(ShenandoahHeapRegion* r); - - bool is_thread_safe() { return true; } -}; - -class ShenandoahCaptureUpdateWaterMarkForOld : public ShenandoahHeapRegionClosure { -private: - ShenandoahMarkingContext* const _ctx; - ShenandoahHeapLock* const _lock; -public: - ShenandoahCaptureUpdateWaterMarkForOld(ShenandoahMarkingContext* ctx); + explicit ShenandoahFinalMarkUpdateRegionStateClosure(ShenandoahMarkingContext* ctx); void heap_region_do(ShenandoahHeapRegion* r); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp index 5010163715cb0..1d3aaed65d1c9 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, Amazon.com, Inc. or its affiliates. All rights reserved. + * Copyright (c) 2022, Amazon.com, Inc. or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -31,64 +31,16 @@ #include "gc/shenandoah/shenandoahOopClosures.inline.hpp" #include "gc/shenandoah/shenandoahGeneration.hpp" #include "gc/shenandoah/shenandoahYoungGeneration.hpp" -#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" -#include "gc/shenandoah/shenandoahWorkerPolicy.hpp" #include "prims/jvmtiTagMap.hpp" #include "utilities/events.hpp" -class ShenandoahConcurrentCoalesceAndFillTask : public WorkerTask { -private: - uint _nworkers; - ShenandoahHeapRegion** _coalesce_and_fill_region_array; - uint _coalesce_and_fill_region_count; - ShenandoahConcurrentGC* _old_gc; - volatile bool _is_preempted; - -public: - ShenandoahConcurrentCoalesceAndFillTask(uint nworkers, ShenandoahHeapRegion** coalesce_and_fill_region_array, - uint region_count, ShenandoahConcurrentGC* old_gc) : - WorkerTask("Shenandoah Concurrent Coalesce and Fill"), - _nworkers(nworkers), - _coalesce_and_fill_region_array(coalesce_and_fill_region_array), - _coalesce_and_fill_region_count(region_count), - _old_gc(old_gc), - _is_preempted(false) { - } - void work(uint worker_id) { - for (uint region_idx = worker_id; region_idx < _coalesce_and_fill_region_count; region_idx += _nworkers) { - ShenandoahHeapRegion* r = _coalesce_and_fill_region_array[region_idx]; - if (!r->is_humongous()) { - if (!r->oop_fill_and_coalesce()) { - // Coalesce and fill has been preempted - Atomic::store(&_is_preempted, true); - return; - } - } else { - // there's only one object in this region and it's not garbage, so no need to coalesce or fill - } - } - } - - // Value returned from is_completed() is only valid after all worker thread have terminated. - bool is_completed() { - return !Atomic::load(&_is_preempted); - } -}; ShenandoahOldGC::ShenandoahOldGC(ShenandoahGeneration* generation, ShenandoahSharedFlag& allow_preemption) : ShenandoahConcurrentGC(generation, false), _allow_preemption(allow_preemption) { - _coalesce_and_fill_region_array = NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, ShenandoahHeap::heap()->num_regions(), mtGC); -} - -void ShenandoahOldGC::start_old_evacuations() { - ShenandoahHeap* heap = ShenandoahHeap::heap(); - ShenandoahOldHeuristics* old_heuristics = heap->old_heuristics(); - old_heuristics->start_old_evacuations(); } - // Final mark for old-gen is different than for young or old, so we // override the implementation. void ShenandoahOldGC::op_final_mark() { @@ -133,56 +85,57 @@ void ShenandoahOldGC::op_final_mark() { bool ShenandoahOldGC::collect(GCCause::Cause cause) { ShenandoahHeap* heap = ShenandoahHeap::heap(); assert(!heap->doing_mixed_evacuations(), "Should not start an old gc with pending mixed evacuations"); + assert(!heap->is_prepare_for_old_mark_in_progress(), "Old regions need to be parseable during concurrent mark."); - if (!heap->is_concurrent_prep_for_mixed_evacuation_in_progress()) { - // Skip over the initial phases of old collect if we're resuming mixed evacuation preparation. - // Continue concurrent mark, do not reset regions, do not mark roots, do not collect $200. - _allow_preemption.set(); - entry_mark(); - if (!_allow_preemption.try_unset()) { - // The regulator thread has unset the preemption guard. That thread will shortly cancel - // the gc, but the control thread is now racing it. Wait until this thread sees the cancellation. - while (!heap->cancelled_gc()) { - SpinPause(); - } - } + // Enable preemption of old generation mark. + _allow_preemption.set(); - if (heap->cancelled_gc()) { - return false; + // Continue concurrent mark, do not reset regions, do not mark roots, do not collect $200. + entry_mark(); + + // If we failed to unset the preemption flag, it means another thread has already unset it. + if (!_allow_preemption.try_unset()) { + // The regulator thread has unset the preemption guard. That thread will shortly cancel + // the gc, but the control thread is now racing it. Wait until this thread sees the + // cancellation. + while (!heap->cancelled_gc()) { + SpinPause(); } + } - // Complete marking under STW - vmop_entry_final_mark(); + if (heap->cancelled_gc()) { + return false; + } - // We aren't dealing with old generation evacuation yet. Our heuristic - // should not have built a cset in final mark. - assert(!heap->is_evacuation_in_progress(), "Old gen evacuations are not supported"); + // Complete marking under STW + vmop_entry_final_mark(); - // Process weak roots that might still point to regions that would be broken by cleanup - if (heap->is_concurrent_weak_root_in_progress()) { - entry_weak_refs(); - entry_weak_roots(); - } + // We aren't dealing with old generation evacuation yet. Our heuristic + // should not have built a cset in final mark. + assert(!heap->is_evacuation_in_progress(), "Old gen evacuations are not supported"); - // Final mark might have reclaimed some immediate garbage, kick cleanup to reclaim - // the space. This would be the last action if there is nothing to evacuate. - entry_cleanup_early(); + // Process weak roots that might still point to regions that would be broken by cleanup + if (heap->is_concurrent_weak_root_in_progress()) { + entry_weak_refs(); + entry_weak_roots(); + } - { - ShenandoahHeapLocker locker(heap->lock()); - heap->free_set()->log_status(); - } + // Final mark might have reclaimed some immediate garbage, kick cleanup to reclaim + // the space. This would be the last action if there is nothing to evacuate. + entry_cleanup_early(); + { + ShenandoahHeapLocker locker(heap->lock()); + heap->free_set()->log_status(); + } - // TODO: Old marking doesn't support class unloading yet - // Perform concurrent class unloading - // if (heap->unload_classes() && - // heap->is_concurrent_weak_root_in_progress()) { - // entry_class_unloading(); - // } - heap->set_concurrent_prep_for_mixed_evacuation_in_progress(true); - } + // TODO: Old marking doesn't support class unloading yet + // Perform concurrent class unloading + // if (heap->unload_classes() && + // heap->is_concurrent_weak_root_in_progress()) { + // entry_class_unloading(); + // } assert(!heap->is_concurrent_strong_root_in_progress(), "No evacuations during old gc."); @@ -193,82 +146,5 @@ bool ShenandoahOldGC::collect(GCCause::Cause cause) { // collection. vmop_entry_final_roots(false); - // Coalesce and fill objects _after_ weak root processing and class unloading. - // Weak root and reference processing makes assertions about unmarked referents - // that will fail if they've been overwritten with filler objects. There is also - // a case in the LRB that permits access to from-space objects for the purpose - // of class unloading that is unlikely to function correctly if the object has - // been filled. - _allow_preemption.set(); - - if (heap->cancelled_gc()) { - return false; - } - - if (heap->is_concurrent_prep_for_mixed_evacuation_in_progress()) { - if (!entry_coalesce_and_fill()) { - // If an allocation failure occurs during coalescing, we will run a degenerated - // cycle for the young generation. This should be a rare event. Normally, we'll - // resume the coalesce-and-fill effort after the preempting young-gen GC finishes. - return false; - } - } - if (!_allow_preemption.try_unset()) { - // The regulator thread has unset the preemption guard. That thread will shortly cancel - // the gc, but the control thread is now racing it. Wait until this thread sees the cancellation. - while (!heap->cancelled_gc()) { - SpinPause(); - } - } - // Prepare for old evacuations (actual evacuations will happen on subsequent young collects). This cannot - // begin until after we have completed coalesce-and-fill. - start_old_evacuations(); - return true; } - -void ShenandoahOldGC::entry_coalesce_and_fill_message(char *buf, size_t len) const { - // ShenandoahHeap* const heap = ShenandoahHeap::heap(); - jio_snprintf(buf, len, "Coalescing and filling (%s)", _generation->name()); -} - -bool ShenandoahOldGC::op_coalesce_and_fill() { - ShenandoahHeap* const heap = ShenandoahHeap::heap(); - ShenandoahOldHeuristics* old_heuristics = heap->old_heuristics(); - WorkerThreads* workers = heap->workers(); - uint nworkers = workers->active_workers(); - - assert(_generation->generation_mode() == OLD, "Only old-GC does coalesce and fill"); - log_debug(gc)("Starting (or resuming) coalesce-and-fill of old heap regions"); - uint coalesce_and_fill_regions_count = old_heuristics->old_coalesce_and_fill_candidates(); - assert(coalesce_and_fill_regions_count <= heap->num_regions(), "Sanity"); - old_heuristics->get_coalesce_and_fill_candidates(_coalesce_and_fill_region_array); - ShenandoahConcurrentCoalesceAndFillTask task(nworkers, _coalesce_and_fill_region_array, coalesce_and_fill_regions_count, this); - - workers->run_task(&task); - if (task.is_completed()) { - // Remember that we're done with coalesce-and-fill. - heap->set_concurrent_prep_for_mixed_evacuation_in_progress(false); - return true; - } else { - log_debug(gc)("Suspending coalesce-and-fill of old heap regions"); - // Otherwise, we got preempted before the work was done. - return false; - } -} - -bool ShenandoahOldGC::entry_coalesce_and_fill() { - char msg[1024]; - ShenandoahHeap* const heap = ShenandoahHeap::heap(); - - entry_coalesce_and_fill_message(msg, sizeof(msg)); - ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::coalesce_and_fill); - - TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); - EventMark em("%s", msg); - ShenandoahWorkerScope scope(heap->workers(), - ShenandoahWorkerPolicy::calc_workers_for_conc_marking(), - "concurrent coalesce and fill"); - - return op_coalesce_and_fill(); -} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.hpp index a6ad266c21856..671ef4f90e70c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.hpp @@ -40,13 +40,8 @@ class ShenandoahOldGC : public ShenandoahConcurrentGC { virtual void op_final_mark(); private: - ShenandoahHeapRegion** _coalesce_and_fill_region_array; - void start_old_evacuations(); - bool entry_coalesce_and_fill(); ShenandoahSharedFlag& _allow_preemption; - bool op_coalesce_and_fill(); - void entry_coalesce_and_fill_message(char *buf, size_t len) const; }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp index 2908c918748a9..951c66d043989 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp @@ -26,9 +26,11 @@ #include "precompiled.hpp" #include "gc/shared/strongRootsScope.hpp" +#include "gc/shenandoah/shenandoahCollectorPolicy.hpp" #include "gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp" #include "gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.hpp" #include "gc/shenandoah/heuristics/shenandoahCompactHeuristics.hpp" +#include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp" #include "gc/shenandoah/heuristics/shenandoahStaticHeuristics.hpp" #include "gc/shenandoah/shenandoahAsserts.hpp" #include "gc/shenandoah/shenandoahFreeSet.hpp" @@ -37,12 +39,17 @@ #include "gc/shenandoah/shenandoahHeapRegion.hpp" #include "gc/shenandoah/shenandoahMarkClosures.hpp" #include "gc/shenandoah/shenandoahMark.inline.hpp" +#include "gc/shenandoah/shenandoahMonitoringSupport.hpp" #include "gc/shenandoah/shenandoahOldGeneration.hpp" #include "gc/shenandoah/shenandoahOopClosures.inline.hpp" #include "gc/shenandoah/shenandoahReferenceProcessor.hpp" #include "gc/shenandoah/shenandoahStringDedup.hpp" #include "gc/shenandoah/shenandoahUtils.hpp" +#include "gc/shenandoah/shenandoahWorkerPolicy.hpp" +#include "gc/shenandoah/shenandoahYoungGeneration.hpp" +#include "prims/jvmtiTagMap.hpp" #include "runtime/threads.hpp" +#include "utilities/events.hpp" class ShenandoahFlushAllSATB : public ThreadClosure { private: @@ -125,8 +132,50 @@ class ShenandoahPurgeSATBTask : public WorkerTask { } }; +class ShenandoahConcurrentCoalesceAndFillTask : public WorkerTask { + private: + uint _nworkers; + ShenandoahHeapRegion** _coalesce_and_fill_region_array; + uint _coalesce_and_fill_region_count; + volatile bool _is_preempted; + + public: + ShenandoahConcurrentCoalesceAndFillTask(uint nworkers, ShenandoahHeapRegion** coalesce_and_fill_region_array, + uint region_count) : + WorkerTask("Shenandoah Concurrent Coalesce and Fill"), + _nworkers(nworkers), + _coalesce_and_fill_region_array(coalesce_and_fill_region_array), + _coalesce_and_fill_region_count(region_count), + _is_preempted(false) { + } + + void work(uint worker_id) { + for (uint region_idx = worker_id; region_idx < _coalesce_and_fill_region_count; region_idx += _nworkers) { + ShenandoahHeapRegion* r = _coalesce_and_fill_region_array[region_idx]; + if (r->is_humongous()) { + // there's only one object in this region and it's not garbage, so no need to coalesce or fill + continue; + } + + if (!r->oop_fill_and_coalesce()) { + // Coalesce and fill has been preempted + Atomic::store(&_is_preempted, true); + return; + } + } + } + + // Value returned from is_completed() is only valid after all worker thread have terminated. + bool is_completed() { + return !Atomic::load(&_is_preempted); + } +}; + ShenandoahOldGeneration::ShenandoahOldGeneration(uint max_queues, size_t max_capacity, size_t soft_max_capacity) - : ShenandoahGeneration(OLD, max_queues, max_capacity, soft_max_capacity) { + : ShenandoahGeneration(OLD, max_queues, max_capacity, soft_max_capacity), + _coalesce_and_fill_region_array(NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, ShenandoahHeap::heap()->num_regions(), mtGC)), + _state(IDLE) +{ // Always clear references for old generation ref_processor()->set_soft_reference_policy(true); } @@ -166,6 +215,66 @@ void ShenandoahOldGeneration::cancel_marking() { ShenandoahGeneration::cancel_marking(); } +void ShenandoahOldGeneration::prepare_gc() { + + // Make the old generation regions parseable, so they can be safely + // scanned when looking for objects in memory indicated by dirty cards. + entry_coalesce_and_fill(); + + // Now that we have made the old generation parseable, it is safe to reset the mark bitmap. + { + static const char* msg = "Concurrent reset (OLD)"; + ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset_old); + ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(), + ShenandoahWorkerPolicy::calc_workers_for_conc_reset(), + msg); + ShenandoahGeneration::prepare_gc(); + } +} + +bool ShenandoahOldGeneration::entry_coalesce_and_fill() { + char msg[1024]; + ShenandoahHeap* const heap = ShenandoahHeap::heap(); + + ShenandoahConcurrentPhase gc_phase("Coalescing and filling (OLD)", ShenandoahPhaseTimings::coalesce_and_fill); + + // TODO: I don't think we're using these concurrent collection counters correctly. + TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); + EventMark em("%s", msg); + ShenandoahWorkerScope scope(heap->workers(), + ShenandoahWorkerPolicy::calc_workers_for_conc_marking(), + "concurrent coalesce and fill"); + + return coalesce_and_fill(); +} + +bool ShenandoahOldGeneration::coalesce_and_fill() { + ShenandoahHeap* const heap = ShenandoahHeap::heap(); + heap->set_prepare_for_old_mark_in_progress(true); + transition_to(FILLING); + + ShenandoahOldHeuristics* old_heuristics = heap->old_heuristics(); + WorkerThreads* workers = heap->workers(); + uint nworkers = workers->active_workers(); + + log_debug(gc)("Starting (or resuming) coalesce-and-fill of old heap regions"); + uint coalesce_and_fill_regions_count = old_heuristics->get_coalesce_and_fill_candidates(_coalesce_and_fill_region_array); + assert(coalesce_and_fill_regions_count <= heap->num_regions(), "Sanity"); + ShenandoahConcurrentCoalesceAndFillTask task(nworkers, _coalesce_and_fill_region_array, coalesce_and_fill_regions_count); + + workers->run_task(&task); + if (task.is_completed()) { + // Remember that we're done with coalesce-and-fill. + heap->set_prepare_for_old_mark_in_progress(false); + transition_to(BOOTSTRAPPING); + return true; + } else { + log_debug(gc)("Suspending coalesce-and-fill of old heap regions"); + // Otherwise, we got preempted before the work was done. + return false; + } +} + void ShenandoahOldGeneration::transfer_pointers_from_satb() { ShenandoahHeap* heap = ShenandoahHeap::heap(); shenandoah_assert_safepoint(); @@ -195,18 +304,121 @@ void ShenandoahOldGeneration::prepare_regions_and_collection_set(bool concurrent } { + // This doesn't actually choose a collection set, but prepares a list of + // regions as 'candidates' for inclusion in a mixed collection. ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::choose_cset : ShenandoahPhaseTimings::degen_gc_choose_cset); ShenandoahHeapLocker locker(heap->lock()); heuristics()->choose_collection_set(nullptr, nullptr); } { + // Though we did not choose a collection set above, we still may have + // freed up immediate garbage regions so proceed with rebuilding the free set. ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_rebuild_freeset : ShenandoahPhaseTimings::degen_gc_final_rebuild_freeset); ShenandoahHeapLocker locker(heap->lock()); heap->free_set()->rebuild(); } } +const char* ShenandoahOldGeneration::state_name(State state) { + switch (state) { + case IDLE: return "Idle"; + case FILLING: return "Coalescing"; + case BOOTSTRAPPING: return "Bootstrapping"; + case MARKING: return "Marking"; + case WAITING: return "Waiting"; + default: + ShouldNotReachHere(); + return "Unknown"; + } +} + +void ShenandoahOldGeneration::transition_to(State new_state) { + if (_state != new_state) { + log_info(gc)("Old generation transition from %s to %s", state_name(_state), state_name(new_state)); + assert(validate_transition(new_state), "Invalid state transition."); + _state = new_state; + } +} + +#ifdef ASSERT +// This diagram depicts the expected state transitions for marking the old generation +// and preparing for old collections. When a young generation cycle executes, the +// remembered set scan must visit objects in old regions. Visiting an object which +// has become dead on previous old cycles will result in crashes. To avoid visiting +// such objects, the remembered set scan will use the old generation mark bitmap when +// possible. It is _not_ possible to use the old generation bitmap when old marking +// is active (bitmap is not complete). For this reason, the old regions are made +// parseable _before_ the old generation bitmap is reset. The diagram does not depict +// global and full collections, both of which cancel any old generation activity. +// +// +-----------------+ +// +------------> | IDLE | +// | +--------> | | +// | | +-----------------+ +// | | | +// | | | Begin Old Mark +// | | v +// | | +-----------------+ +--------------------+ +// | | | FILLING | <-> | YOUNG GC | +// | | | | | (RSet Uses Bitmap) | +// | | +-----------------+ +--------------------+ +// | | | +// | | | Reset Bitmap +// | | v +// | | +-----------------+ +// | | | BOOTSTRAP | +// | | | | +// | | +-----------------+ +// | | | +// | | | Continue Marking +// | | v +// | | +-----------------+ +----------------------+ +// | | | MARKING | <-> | YOUNG GC | +// | +----------| | | (RSet Parses Region) | +// | +-----------------+ +----------------------+ +// | | +// | | Has Candidates +// | v +// | +-----------------+ +// | | WAITING | +// +------------- | | +// +-----------------+ +// +bool ShenandoahOldGeneration::validate_transition(State new_state) { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + switch (new_state) { + case IDLE: + assert(!heap->is_concurrent_old_mark_in_progress(), "Cannot become idle during old mark."); + assert(_old_heuristics->unprocessed_old_collection_candidates() == 0, "Cannot become idle with collection candidates"); + assert(!heap->is_prepare_for_old_mark_in_progress(), "Cannot become idle while making old generation parseable."); + assert(heap->young_generation()->old_gen_task_queues() == nullptr, "Cannot become idle when setup for bootstrapping."); + return true; + case FILLING: + assert(_state == IDLE, "Cannot begin filling without first being idle."); + assert(heap->is_prepare_for_old_mark_in_progress(), "Should be preparing for old mark now."); + return true; + case BOOTSTRAPPING: + assert(_state == FILLING, "Cannot reset bitmap without making old regions parseable."); + // assert(heap->young_generation()->old_gen_task_queues() != nullptr, "Cannot bootstrap without old mark queues."); + assert(!heap->is_prepare_for_old_mark_in_progress(), "Cannot still be making old regions parseable."); + return true; + case MARKING: + assert(_state == BOOTSTRAPPING, "Must have finished bootstrapping before marking."); + assert(heap->young_generation()->old_gen_task_queues() != nullptr, "Young generation needs old mark queues."); + assert(heap->is_concurrent_old_mark_in_progress(), "Should be marking old now."); + return true; + case WAITING: + assert(_state == MARKING, "Cannot have old collection candidates without first marking."); + assert(_old_heuristics->unprocessed_old_collection_candidates() > 0, "Must have collection candidates here."); + return true; + default: + ShouldNotReachHere(); + return false; + } +} +#endif + ShenandoahHeuristics* ShenandoahOldGeneration::initialize_heuristics(ShenandoahMode* gc_mode) { assert(ShenandoahOldGCHeuristics != NULL, "ShenandoahOldGCHeuristics should not equal NULL"); ShenandoahHeuristics* trigger; @@ -222,6 +434,12 @@ ShenandoahHeuristics* ShenandoahOldGeneration::initialize_heuristics(ShenandoahM return NULL; } trigger->set_guaranteed_gc_interval(ShenandoahGuaranteedOldGCInterval); - _heuristics = new ShenandoahOldHeuristics(this, trigger); + _old_heuristics = new ShenandoahOldHeuristics(this, trigger); + _heuristics = _old_heuristics; return _heuristics; } + +void ShenandoahOldGeneration::record_success_concurrent(bool abbreviated) { + heuristics()->record_success_concurrent(abbreviated); + ShenandoahHeap::heap()->shenandoah_policy()->record_success_old(); +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp index 39e2656e2deb3..618a664ce4672 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp @@ -29,6 +29,7 @@ class ShenandoahHeapRegion; class ShenandoahHeapRegionClosure; +class ShenandoahOldHeuristics; class ShenandoahOldGeneration : public ShenandoahGeneration { public: @@ -48,6 +49,8 @@ class ShenandoahOldGeneration : public ShenandoahGeneration { virtual void cancel_marking() override; + virtual void prepare_gc() override; + void prepare_regions_and_collection_set(bool concurrent) override; virtual ShenandoahHeuristics* initialize_heuristics(ShenandoahMode* gc_mode) override; @@ -73,6 +76,32 @@ class ShenandoahOldGeneration : public ShenandoahGeneration { void transfer_pointers_from_satb(); bool is_concurrent_mark_in_progress() override; + + virtual void record_success_concurrent(bool abbreviated) override; + + enum State { + IDLE, FILLING, BOOTSTRAPPING, MARKING, WAITING + }; + + static const char* state_name(State state); + + void transition_to(State new_state); + +#ifdef ASSERT + bool validate_transition(State new_state); +#endif + + State state() const { + return _state; + } + + private: + bool entry_coalesce_and_fill(); + bool coalesce_and_fill(); + + ShenandoahHeapRegion** _coalesce_and_fill_region_array; + ShenandoahOldHeuristics* _old_heuristics; + State _state; }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp index 250eb40d93dd3..35ebf73327db5 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp @@ -49,7 +49,7 @@ class outputStream; #define SHENANDOAH_PHASE_DO(f) \ f(conc_reset, "Concurrent Reset") \ - \ + f(conc_reset_old, "Concurrent Reset (OLD)") \ f(init_mark_gross, "Pause Init Mark (G)") \ f(init_mark, "Pause Init Mark (N)") \ f(init_manage_tlabs, " Manage TLABs") \ @@ -102,7 +102,7 @@ class outputStream; f(conc_strong_roots, "Concurrent Strong Roots") \ SHENANDOAH_PAR_PHASE_DO(conc_strong_roots_, " CSR: ", f) \ f(coalesce_and_fill, "Coalesce and Fill Old Dead") \ - SHENANDOAH_PAR_PHASE_DO(coalesce_and_fill_, " CFOD: ", f) \ + SHENANDOAH_PAR_PHASE_DO(coalesce_and_fill_, " CFOD: ", f) \ f(conc_evac, "Concurrent Evacuation") \ \ f(final_roots_gross, "Pause Final Roots (G)") \ diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.cpp index c193b46aa3845..2891edb3a2c10 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.cpp @@ -26,8 +26,8 @@ #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" #include "gc/shenandoah/mode/shenandoahMode.hpp" #include "gc/shenandoah/shenandoahControlThread.hpp" -#include "gc/shenandoah/shenandoahGeneration.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" +#include "gc/shenandoah/shenandoahOldGeneration.hpp" #include "gc/shenandoah/shenandoahRegulatorThread.hpp" #include "gc/shenandoah/shenandoahYoungGeneration.hpp" #include "logging/log.hpp" @@ -82,7 +82,7 @@ void ShenandoahRegulatorThread::regulate_concurrent_cycles() { log_info(gc)("Heuristics request for young collection accepted"); } } - } else if (mode == ShenandoahControlThread::marking_old) { + } else if (mode == ShenandoahControlThread::servicing_old) { if (start_young_cycle()) { log_info(gc)("Heuristics request to interrupt old for young collection accepted"); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp index 5359def6cc105..5beffc245931f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp @@ -514,7 +514,7 @@ ShenandoahScanRemembered::process_clusters(size_t first_cluster, ShenandoahHeap* heap = ShenandoahHeap::heap(); ShenandoahMarkingContext* ctx; - if (heap->doing_mixed_evacuations() || heap->is_concurrent_prep_for_mixed_evacuation_in_progress()) { + if (heap->is_old_bitmap_stable()) { ctx = heap->marking_context(); } else { ctx = nullptr; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp b/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp index 08385f6f5fe90..4fbdd69ac9f93 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp @@ -31,8 +31,8 @@ #include "gc/shared/referenceProcessorStats.hpp" #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" #include "gc/shenandoah/shenandoahCollectorPolicy.hpp" -#include "gc/shenandoah/shenandoahGeneration.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" +#include "gc/shenandoah/shenandoahOldGeneration.hpp" #include "gc/shenandoah/shenandoahReferenceProcessor.hpp" #include "gc/shenandoah/shenandoahUtils.hpp" #include "gc/shenandoah/shenandoahYoungGeneration.hpp" diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp index e19e020a3e120..a88449e9ee490 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp @@ -60,12 +60,10 @@ class VM_ShenandoahReferenceOperation : public VM_ShenandoahOperation { class VM_ShenandoahInitMark: public VM_ShenandoahOperation { private: ShenandoahConcurrentGC* const _gc; - const bool _do_old_gc_bootstrap; public: - VM_ShenandoahInitMark(ShenandoahConcurrentGC* gc, bool do_old_gc_bootstrap) : + explicit VM_ShenandoahInitMark(ShenandoahConcurrentGC* gc) : VM_ShenandoahOperation(), - _gc(gc), - _do_old_gc_bootstrap(do_old_gc_bootstrap) {}; + _gc(gc) {}; VM_Operation::VMOp_Type type() const { return VMOp_ShenandoahInitMark; } const char* name() const { return "Shenandoah Init Marking"; } virtual void doit(); @@ -75,7 +73,7 @@ class VM_ShenandoahFinalMarkStartEvac: public VM_ShenandoahOperation { private: ShenandoahConcurrentGC* const _gc; public: - VM_ShenandoahFinalMarkStartEvac(ShenandoahConcurrentGC* gc) : + explicit VM_ShenandoahFinalMarkStartEvac(ShenandoahConcurrentGC* gc) : VM_ShenandoahOperation(), _gc(gc) {}; VM_Operation::VMOp_Type type() const { return VMOp_ShenandoahFinalMarkStartEvac; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp index abe619aa263be..2c4f0003f733c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp @@ -43,8 +43,8 @@ const char* ShenandoahYoungGeneration::name() const { void ShenandoahYoungGeneration::set_concurrent_mark_in_progress(bool in_progress) { ShenandoahHeap* heap = ShenandoahHeap::heap(); heap->set_concurrent_young_mark_in_progress(in_progress); - if (_old_gen_task_queues != nullptr && in_progress && !heap->is_concurrent_prep_for_mixed_evacuation_in_progress()) { - // This is not a bug. When the young generation marking is complete, + if (_old_gen_task_queues != nullptr && in_progress && !heap->is_prepare_for_old_mark_in_progress()) { + // This is not a bug. When the bootstrapping marking phase is complete, // the old generation marking is still in progress, unless it's not. // In the case that old-gen preparation for mixed evacuation has been // preempted, we do not want to set concurrent old mark to be in progress. diff --git a/test/hotspot/gtest/gc/shenandoah/test_shenandoahOldHeuristic.cpp b/test/hotspot/gtest/gc/shenandoah/test_shenandoahOldHeuristic.cpp new file mode 100644 index 0000000000000..850a8b63b4d96 --- /dev/null +++ b/test/hotspot/gtest/gc/shenandoah/test_shenandoahOldHeuristic.cpp @@ -0,0 +1,273 @@ +#include "precompiled.hpp" +#include "unittest.hpp" +#include "gc/shenandoah/shenandoahHeap.inline.hpp" +#include "gc/shenandoah/shenandoahHeapRegion.hpp" +#include "gc/shenandoah/shenandoahGeneration.hpp" +#include "gc/shenandoah/shenandoahOldGeneration.hpp" +#include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp" + +// These tests will all be skipped (unless Shenandoah becomes the default +// collector). To execute these tests, you must enable Shenandoah, which +// is done with: +// +// % _JAVA_OPTIONS="-XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational" make exploded-test TEST="gtest:Shenandoah*" +// +// Please note that these 'unit' tests are really integration tests and rely +// on the JVM being initialized. These tests manipulate the state of the +// collector in ways that are not compatible with a normal collection run. +// If these tests take longer than the minimum time between gc intervals - +// or, more likely, if you have them paused in a debugger longer than this +// interval - you can expect trouble. + +#define SKIP_IF_NOT_SHENANDOAH() \ + if (!UseShenandoahGC) { \ + tty->print_cr("skipped"); \ + return; \ + } + +class ShenandoahResetRegions : public ShenandoahHeapRegionClosure { + public: + virtual void heap_region_do(ShenandoahHeapRegion* region) override { + if (!region->is_empty()) { + region->make_trash(); + region->make_empty(); + } + region->set_affiliation(FREE); + region->clear_live_data(); + region->set_top(region->bottom()); + } +}; + +class ShenandoahOldHeuristicTest : public ::testing::Test { + protected: + ShenandoahHeap* _heap; + ShenandoahOldHeuristics* _heuristics; + ShenandoahCollectionSet* _collection_set; + + ShenandoahOldHeuristicTest() + : _heap(ShenandoahHeap::heap()), + _heuristics(_heap->old_heuristics()), + _collection_set(_heap->collection_set()) { + ShenandoahHeapLocker locker(_heap->lock()); + ShenandoahResetRegions reset; + _heap->heap_region_iterate(&reset); + _heap->set_old_evac_reserve(_heap->old_generation()->soft_max_capacity() / 4); + _heuristics->abandon_collection_candidates(); + _collection_set->clear(); + } + + size_t make_garbage(size_t region_idx, size_t garbage_bytes) { + ShenandoahHeapLocker locker(_heap->lock()); + ShenandoahHeapRegion* region = _heap->get_region(region_idx); + region->make_regular_allocation(OLD_GENERATION); + region->increase_live_data_alloc_words(1); + region->set_top(region->bottom() + garbage_bytes / HeapWordSize); + return region->garbage(); + } + + size_t create_too_much_garbage_for_one_mixed_evacuation() { + size_t garbage_target = _heap->old_generation()->soft_max_capacity() / 2; + size_t garbage_total = 0; + size_t region_idx = 0; + while (garbage_total < garbage_target && region_idx < _heap->num_regions()) { + garbage_total += make_garbage_above_threshold(region_idx++); + } + return garbage_total; + } + + void make_pinned(size_t region_idx) { + ShenandoahHeapLocker locker(_heap->lock()); + ShenandoahHeapRegion* region = _heap->get_region(region_idx); + region->record_pin(); + region->make_pinned(); + } + + void make_unpinned(size_t region_idx) { + ShenandoahHeapLocker locker(_heap->lock()); + ShenandoahHeapRegion* region = _heap->get_region(region_idx); + region->record_unpin(); + region->make_unpinned(); + } + + size_t make_garbage_below_threshold(size_t region_idx) { + return make_garbage(region_idx, collection_threshold() - 100); + } + + size_t make_garbage_above_threshold(size_t region_idx) { + return make_garbage(region_idx, collection_threshold() + 100); + } + + size_t collection_threshold() const { + return ShenandoahHeapRegion::region_size_bytes() * ShenandoahOldGarbageThreshold / 100; + } +}; + +TEST_VM_F(ShenandoahOldHeuristicTest, select_no_old_regions) { + SKIP_IF_NOT_SHENANDOAH(); + + _heuristics->prepare_for_old_collections(); + EXPECT_EQ(0U, _heuristics->last_old_region_index()); + EXPECT_EQ(0U, _heuristics->last_old_collection_candidate_index()); + EXPECT_EQ(0U, _heuristics->unprocessed_old_collection_candidates()); +} + +TEST_VM_F(ShenandoahOldHeuristicTest, select_no_old_region_above_threshold) { + SKIP_IF_NOT_SHENANDOAH(); + + // In this case, we have zero regions to add to the collection set, + // but we will have one region that must still be made parseable. + make_garbage_below_threshold(10); + _heuristics->prepare_for_old_collections(); + EXPECT_EQ(1U, _heuristics->last_old_region_index()); + EXPECT_EQ(0U, _heuristics->last_old_collection_candidate_index()); + EXPECT_EQ(0U, _heuristics->unprocessed_old_collection_candidates()); +} + +TEST_VM_F(ShenandoahOldHeuristicTest, select_one_old_region_above_threshold) { + SKIP_IF_NOT_SHENANDOAH(); + + make_garbage_above_threshold(10); + _heuristics->prepare_for_old_collections(); + EXPECT_EQ(1U, _heuristics->last_old_region_index()); + EXPECT_EQ(1U, _heuristics->last_old_collection_candidate_index()); + EXPECT_EQ(1U, _heuristics->unprocessed_old_collection_candidates()); +} + +TEST_VM_F(ShenandoahOldHeuristicTest, prime_one_old_region) { + SKIP_IF_NOT_SHENANDOAH(); + + size_t garbage = make_garbage_above_threshold(10); + _heuristics->prepare_for_old_collections(); + _heuristics->prime_collection_set(_collection_set); + + EXPECT_EQ(garbage, _collection_set->get_old_garbage()); + EXPECT_EQ(0U, _heuristics->unprocessed_old_collection_candidates()); +} + +TEST_VM_F(ShenandoahOldHeuristicTest, prime_many_old_regions) { + SKIP_IF_NOT_SHENANDOAH(); + + size_t g1 = make_garbage_above_threshold(100); + size_t g2 = make_garbage_above_threshold(101); + _heuristics->prepare_for_old_collections(); + _heuristics->prime_collection_set(_collection_set); + + EXPECT_EQ(g1 + g2, _collection_set->get_old_garbage()); + EXPECT_EQ(0U, _heuristics->unprocessed_old_collection_candidates()); +} + +TEST_VM_F(ShenandoahOldHeuristicTest, require_multiple_mixed_evacuations) { + SKIP_IF_NOT_SHENANDOAH(); + + size_t garbage = create_too_much_garbage_for_one_mixed_evacuation(); + _heuristics->prepare_for_old_collections(); + _heuristics->prime_collection_set(_collection_set); + + EXPECT_LT(_collection_set->get_old_garbage(), garbage); + EXPECT_GT(_heuristics->unprocessed_old_collection_candidates(), 0UL); +} + +TEST_VM_F(ShenandoahOldHeuristicTest, skip_pinned_regions) { + SKIP_IF_NOT_SHENANDOAH(); + + // Create three old regions with enough garbage to be collected. + size_t g1 = make_garbage_above_threshold(1); + size_t g2 = make_garbage_above_threshold(2); + size_t g3 = make_garbage_above_threshold(3); + + // A region can be pinned when we chose collection set candidates. + make_pinned(2); + _heuristics->prepare_for_old_collections(); + + // We only excluded pinned regions when we actually add regions to the collection set. + ASSERT_EQ(3UL, _heuristics->unprocessed_old_collection_candidates()); + + // Here the region is still pinned, so it cannot be added to the collection set. + _heuristics->prime_collection_set(_collection_set); + + // The two unpinned regions should be added to the collection set and the pinned + // region should be retained at the front of the list of candidates as it would be + // likely to become unpinned by the next mixed collection cycle. + EXPECT_EQ(_collection_set->get_old_garbage(), g1 + g3); + EXPECT_EQ(_heuristics->unprocessed_old_collection_candidates(), 1UL); + + // Simulate another mixed collection after making region 2 unpinned. This time, + // the now unpinned region should be added to the collection set. + make_unpinned(2); + _collection_set->clear(); + _heuristics->prime_collection_set(_collection_set); + + EXPECT_EQ(_collection_set->get_old_garbage(), g2); + EXPECT_EQ(_heuristics->unprocessed_old_collection_candidates(), 0UL); +} + +TEST_VM_F(ShenandoahOldHeuristicTest, pinned_region_is_first) { + SKIP_IF_NOT_SHENANDOAH(); + + // Create three old regions with enough garbage to be collected. + size_t g1 = make_garbage_above_threshold(1); + size_t g2 = make_garbage_above_threshold(2); + size_t g3 = make_garbage_above_threshold(3); + + make_pinned(1); + _heuristics->prepare_for_old_collections(); + _heuristics->prime_collection_set(_collection_set); + + EXPECT_EQ(_collection_set->get_old_garbage(), g2 + g3); + EXPECT_EQ(_heuristics->unprocessed_old_collection_candidates(), 1UL); + + make_unpinned(1); + _collection_set->clear(); + _heuristics->prime_collection_set(_collection_set); + + EXPECT_EQ(_collection_set->get_old_garbage(), g1); + EXPECT_EQ(_heuristics->unprocessed_old_collection_candidates(), 0UL); +} + +TEST_VM_F(ShenandoahOldHeuristicTest, pinned_region_is_last) { + SKIP_IF_NOT_SHENANDOAH(); + + // Create three old regions with enough garbage to be collected. + size_t g1 = make_garbage_above_threshold(1); + size_t g2 = make_garbage_above_threshold(2); + size_t g3 = make_garbage_above_threshold(3); + + make_pinned(3); + _heuristics->prepare_for_old_collections(); + _heuristics->prime_collection_set(_collection_set); + + EXPECT_EQ(_collection_set->get_old_garbage(), g1 + g2); + EXPECT_EQ(_heuristics->unprocessed_old_collection_candidates(), 1UL); + + make_unpinned(3); + _collection_set->clear(); + _heuristics->prime_collection_set(_collection_set); + + EXPECT_EQ(_collection_set->get_old_garbage(), g3); + EXPECT_EQ(_heuristics->unprocessed_old_collection_candidates(), 0UL); +} + +TEST_VM_F(ShenandoahOldHeuristicTest, unpinned_region_is_middle) { + SKIP_IF_NOT_SHENANDOAH(); + + // Create three old regions with enough garbage to be collected. + size_t g1 = make_garbage_above_threshold(1); + size_t g2 = make_garbage_above_threshold(2); + size_t g3 = make_garbage_above_threshold(3); + + make_pinned(1); + make_pinned(3); + _heuristics->prepare_for_old_collections(); + _heuristics->prime_collection_set(_collection_set); + + EXPECT_EQ(_collection_set->get_old_garbage(), g2); + EXPECT_EQ(_heuristics->unprocessed_old_collection_candidates(), 2UL); + + make_unpinned(1); + make_unpinned(3); + _collection_set->clear(); + _heuristics->prime_collection_set(_collection_set); + + EXPECT_EQ(_collection_set->get_old_garbage(), g1 + g3); + EXPECT_EQ(_heuristics->unprocessed_old_collection_candidates(), 0UL); +} From b4ff060bf879a56141bd7c46713675ae2f2b466f Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Sat, 13 Aug 2022 00:15:51 +0000 Subject: [PATCH 145/254] Fix budgeting assertion Reviewed-by: wkemper --- .../shenandoahAdaptiveHeuristics.cpp | 65 ++++++++++++++----- .../gc/shenandoah/shenandoahCollectionSet.cpp | 3 +- .../gc/shenandoah/shenandoahGeneration.cpp | 26 ++++++-- 3 files changed, 70 insertions(+), 24 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp index af3bb6e153e29..e07315e1555d9 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp @@ -94,6 +94,10 @@ void ShenandoahAdaptiveHeuristics::choose_collection_set_from_regiondata(Shenand bool is_generational = heap->mode()->is_generational(); bool is_global = (_generation->generation_mode() == GLOBAL); size_t capacity = heap->young_generation()->max_capacity(); + + // cur_young_garbage represents the amount of memory to be reclaimed from young-gen. In the case that live objects + // are known to be promoted out of young-gen, we count this as cur_young_garbage because this memory is reclaimed + // from young-gen and becomes available to serve future young-gen allocation requests. size_t cur_young_garbage = 0; // Better select garbage-first regions @@ -123,12 +127,18 @@ void ShenandoahAdaptiveHeuristics::choose_collection_set_from_regiondata(Shenand add_region = true; old_cur_cset = new_cset; } - } else if (r->age() >= InitialTenuringThreshold) { + } else if (cset->is_preselected(r->index())) { + assert(r->age() >= InitialTenuringThreshold, "Preselected regions must have tenure age"); // Entire region will be promoted, This region does not impact young-gen or old-gen evacuation reserve. // This region has been pre-selected and its impact on promotion reserve is already accounted for. add_region = true; - cur_young_garbage += r->garbage(); - } else { + // r->used() is r->garbage() + r->get_live_data_bytes() + // Since all live data in this region is being evacuated from young-gen, it is as if this memory + // is garbage insofar as young-gen is concerned. Counting this as garbage reduces the need to + // reclaim highly utilized young-gen regions just for the sake of finding min_garbage to reclaim + // within youn-gen memory. + cur_young_garbage += r->used(); + } else if (r->age() < InitialTenuringThreshold) { size_t new_cset = young_cur_cset + r->get_live_data_bytes(); size_t region_garbage = r->garbage(); size_t new_garbage = cur_young_garbage + region_garbage; @@ -139,13 +149,16 @@ void ShenandoahAdaptiveHeuristics::choose_collection_set_from_regiondata(Shenand cur_young_garbage = new_garbage; } } + // Note that we do not add aged regions if they were not pre-selected. The reason they were not preselected + // is because there is not sufficient room in old-gen to hold their to-be-promoted live objects. if (add_region) { cset->add_region(r); } } } else { - // This is young-gen collection. + // This is young-gen collection or a mixed evacuation. If this is mixed evacuation, the old-gen candidate regions + // have already been added. size_t max_cset = (size_t) (heap->get_young_evac_reserve() / ShenandoahEvacWaste); size_t cur_cset = 0; size_t free_target = (capacity * ShenandoahMinFreeThreshold) / 100 + max_cset; @@ -160,19 +173,37 @@ void ShenandoahAdaptiveHeuristics::choose_collection_set_from_regiondata(Shenand size_t new_cset; size_t region_garbage = r->garbage(); size_t new_garbage = cur_young_garbage + region_garbage; - if (r->age() >= InitialTenuringThreshold) { - // Entire region will be promoted, This region does not impact young-gen evacuation reserve. Memory has already - // been set aside to hold evacuation results as advance_promotion_reserve. - new_cset = cur_cset; - } else { - new_cset = cur_cset + r->get_live_data_bytes(); - } - bool add_regardless = (region_garbage > ignore_threshold) && (new_garbage < min_garbage); - if ((new_cset <= max_cset) && - (add_regardless || (region_garbage > garbage_threshold) || (r->age() >= InitialTenuringThreshold))) { - cset->add_region(r); - cur_cset = new_cset; - cur_young_garbage = new_garbage; + bool add_region = false; + + if (!r->is_old()) { + if (cset->is_preselected(r->index())) { + assert(r->age() >= InitialTenuringThreshold, "Preselected regions must have tenure age"); + // Entire region will be promoted, This region does not impact young-gen evacuation reserve. Memory has already + // been set aside to hold evacuation results as advance_promotion_reserve. + add_region = true; + new_cset = cur_cset; + // Since all live data in this region is being evacuated from young-gen, it is as if this memory + // is garbage insofar as young-gen is concerned. Counting this as garbage reduces the need to + // reclaim highly utilized young-gen regions just for the sake of finding min_garbage to reclaim + // within youn-gen memory + cur_young_garbage += r->get_live_data_bytes(); + } else if (r->age() < InitialTenuringThreshold) { + new_cset = cur_cset + r->get_live_data_bytes(); + size_t region_garbage = r->garbage(); + size_t new_garbage = cur_young_garbage + region_garbage; + bool add_regardless = (region_garbage > ignore_threshold) && (new_garbage < min_garbage); + if ((new_cset <= max_cset) && (add_regardless || (region_garbage > garbage_threshold))) { + add_region = true; + cur_cset = new_cset; + cur_young_garbage = new_garbage; + } + } + // Note that we do not add aged regions if they were not pre-selected. The reason they were not preselected + // is because there is not sufficient room in old-gen to hold their to-be-promoted live objects. + + if (add_region) { + cset->add_region(r); + } } } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp index d23586b64f2f7..c828dfd137676 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp @@ -127,9 +127,10 @@ void ShenandoahCollectionSet::clear() { _current_index = 0; _young_region_count = 0; - _old_region_count = 0; _young_bytes_to_evacuate = 0; _young_bytes_to_promote = 0; + + _old_region_count = 0; _old_bytes_to_evacuate = 0; _has_old_regions = false; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index 3502288b06144..e310a8ffe298a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -430,6 +430,7 @@ void ShenandoahGeneration::adjust_evacuation_budgets(ShenandoahHeap* heap, Shena consumed_by_advance_promotion, (size_t) (collection_set->get_young_bytes_to_be_promoted() * ShenandoahEvacWaste)); collection_set->abandon_preselected(); + if (old_evacuated_committed > old_evacuation_reserve) { // This should only happen due to round-off errors when enforcing ShenandoahEvacWaste assert(old_evacuated_committed <= (33 * old_evacuation_reserve) / 32, @@ -438,6 +439,11 @@ void ShenandoahGeneration::adjust_evacuation_budgets(ShenandoahHeap* heap, Shena old_evacuated_committed = old_evacuation_reserve; } else if (old_evacuated_committed < old_evacuation_reserve) { // This may happen if the old-gen collection consumes less than full budget. + + // If we shrink old_evacuation_reserve by more than a region size, we can expand regions_available_to_loan. + // Can only give back regions that are fully unused, so round down. + size_t old_evac_regions_unused = (old_evacuation_reserve - old_evacuated_committed) / region_size_bytes; + regions_available_to_loan += old_evac_regions_unused; old_evacuation_reserve = old_evacuated_committed; heap->set_old_evac_reserve(old_evacuation_reserve); } @@ -462,9 +468,10 @@ void ShenandoahGeneration::adjust_evacuation_budgets(ShenandoahHeap* heap, Shena // Undo the previous loan regions_available_to_loan += old_regions_loaned_for_young_evac; old_regions_loaned_for_young_evac = revised_loan_for_young_evacuation; + // And make a new loan assert(regions_available_to_loan > old_regions_loaned_for_young_evac, "Cannot loan regions that we do not have"); - regions_available_to_loan -= old_regions_loaned_for_young_evac; + regions_available_to_loan -= revised_loan_for_young_evacuation; } else { // Undo the prevous loan regions_available_to_loan += old_regions_loaned_for_young_evac; @@ -496,8 +503,9 @@ void ShenandoahGeneration::adjust_evacuation_budgets(ShenandoahHeap* heap, Shena // "all the rest" of old-gen memory into the promotion reserve, we'll have nothing left to loan to young-gen // during the evac and update phases of GC. So we "limit" the sizes of the promotion budget to be the smaller of: // - // 1. old_gen->available - (old_evacuation_committed + old_bytes_loaned_for_young_evac + consumed_by_advance_promotion) - // 2. young bytes reserved for evacuation + // 1. old_gen->available - (old_evacuation_committed + old_bytes_loaned_for_young_evac + consumed_by_advance_promotion + // + old_bytes_[already]_ reserved_for_alloc_supplement) + // 2. young bytes reserved for evacuation (we can't promote more than young is evacuating) assert(old_available > old_evacuated_committed, "Cannot evacuate more than available"); assert(old_available > old_evacuated_committed + old_bytes_loaned_for_young_evac, @@ -508,8 +516,10 @@ void ShenandoahGeneration::adjust_evacuation_budgets(ShenandoahHeap* heap, Shena consumed_by_advance_promotion + old_bytes_reserved_for_alloc_supplement), "Cannot loan for alloc supplement more than available"); - size_t promotion_reserve = old_available - (old_evacuated_committed + consumed_by_advance_promotion + - old_bytes_loaned_for_young_evac + old_bytes_reserved_for_alloc_supplement); + size_t promotion_reserve = regions_available_to_loan * region_size_bytes; + assert(promotion_reserve <= old_available - (old_evacuated_committed + consumed_by_advance_promotion + + old_bytes_loaned_for_young_evac + old_bytes_reserved_for_alloc_supplement), + "Byte reserves do not match region reserves"); // We experimented with constraining promoted_reserve to be no larger than 4 times the size of previously_promoted, // but this constraint was too limiting, resulting in failure of legitimate promotions. This was tried before we @@ -544,6 +554,10 @@ void ShenandoahGeneration::adjust_evacuation_budgets(ShenandoahHeap* heap, Shena promotion_reserve += consumed_by_advance_promotion; heap->set_promoted_reserve(promotion_reserve); + + size_t promotion_regions = (promotion_reserve + region_size_bytes - 1) / region_size_bytes; + assert(regions_available_to_loan >= promotion_regions, "Promoting more regions than memory is available"); + regions_available_to_loan -= promotion_regions; heap->reset_promoted_expended(); if (collection_set->get_old_bytes_reserved_for_evacuation() == 0) { // Setting old evacuation reserve to zero denotes that there is no old-gen evacuation in this pass. @@ -607,7 +621,7 @@ void ShenandoahGeneration::adjust_evacuation_budgets(ShenandoahHeap* heap, Shena byte_size_in_proper_unit(old_evacuated), proper_unit_for_byte_size(old_evacuated), byte_size_in_proper_unit(old_available), proper_unit_for_byte_size(old_available)); - assert(old_available > old_evacuation_reserve + promotion_reserve + old_bytes_loaned_for_young_evac + allocation_supplement, + assert(old_available >= old_evacuation_reserve + promotion_reserve + old_bytes_loaned_for_young_evac + allocation_supplement, "old_available must be larger than accumulated reserves"); size_t regular_promotion = promotion_reserve - consumed_by_advance_promotion; From 270e8b89a1599c8cb87f9846fb0eb146ecd525b6 Mon Sep 17 00:00:00 2001 From: "Y. Srinivas Ramakrishna" Date: Mon, 29 Aug 2022 18:09:58 +0000 Subject: [PATCH 146/254] Use only up to ConcGCThreads for concurrent RS scanning. Reviewed-by: kdnilsen, wkemper --- .../gc/shenandoah/shenandoahConcurrentGC.cpp | 24 +++++++++++++++---- .../gc/shenandoah/shenandoahConcurrentGC.hpp | 1 + .../gc/shenandoah/shenandoahGeneration.cpp | 1 + .../gc/shenandoah/shenandoahWorkerPolicy.cpp | 10 ++++++++ .../gc/shenandoah/shenandoahWorkerPolicy.hpp | 4 ++++ 5 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index b836a551eb7da..824023942ac13 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -121,10 +121,9 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) { TASKQUEUE_STATS_ONLY(_mark.task_queues()->reset_taskqueue_stats()); // Concurrent remembered set scanning - if (_generation->generation_mode() == YOUNG) { - ShenandoahConcurrentPhase gc_phase("Concurrent remembered set scanning", ShenandoahPhaseTimings::init_scan_rset); - _generation->scan_remembered_set(true /* is_concurrent */); - } + entry_scan_remembered_set(); + // When RS scanning yields, we will need a check_cancellation_and_abort() + // degeneration point here. // Concurrent mark roots entry_mark_roots(); @@ -373,6 +372,23 @@ void ShenandoahConcurrentGC::entry_reset() { op_reset(); } +void ShenandoahConcurrentGC::entry_scan_remembered_set() { + if (_generation->generation_mode() == YOUNG) { + ShenandoahHeap* const heap = ShenandoahHeap::heap(); + TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); + const char* msg = "Concurrent remembered set scanning"; + ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::init_scan_rset); + EventMark em("%s", msg); + + ShenandoahWorkerScope scope(heap->workers(), + ShenandoahWorkerPolicy::calc_workers_for_rs_scanning(), + msg); + + heap->try_inject_alloc_failure(); + _generation->scan_remembered_set(true /* is_concurrent */); + } +} + void ShenandoahConcurrentGC::entry_mark_roots() { ShenandoahHeap* const heap = ShenandoahHeap::heap(); TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp index b6c8b2a4d1626..6d431c645fd51 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp @@ -86,6 +86,7 @@ class ShenandoahConcurrentGC : public ShenandoahGC { // for concurrent operation. void entry_reset(); void entry_mark_roots(); + void entry_scan_remembered_set(); protected: void entry_mark(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index e310a8ffe298a..19c63af676128 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -800,6 +800,7 @@ void ShenandoahGeneration::scan_remembered_set(bool is_concurrent) { ShenandoahReferenceProcessor* rp = ref_processor(); ShenandoahRegionChunkIterator work_list(nworkers); ShenandoahScanRememberedTask task(task_queues(), old_gen_task_queues(), rp, &work_list, is_concurrent); + heap->assert_gc_workers(nworkers); heap->workers()->run_task(&task); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.cpp b/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.cpp index 5c06bdbf9b420..3ea4e17ca60fd 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.cpp @@ -32,6 +32,7 @@ uint ShenandoahWorkerPolicy::_prev_par_marking = 0; uint ShenandoahWorkerPolicy::_prev_conc_marking = 0; +uint ShenandoahWorkerPolicy::_prev_conc_rs_scanning = 0; uint ShenandoahWorkerPolicy::_prev_conc_evac = 0; uint ShenandoahWorkerPolicy::_prev_conc_root_proc = 0; uint ShenandoahWorkerPolicy::_prev_conc_refs_proc = 0; @@ -61,6 +62,15 @@ uint ShenandoahWorkerPolicy::calc_workers_for_conc_marking() { return _prev_conc_marking; } +uint ShenandoahWorkerPolicy::calc_workers_for_rs_scanning() { + uint active_workers = (_prev_conc_rs_scanning == 0) ? ConcGCThreads : _prev_conc_rs_scanning; + _prev_conc_rs_scanning = + WorkerPolicy::calc_active_conc_workers(ConcGCThreads, + active_workers, + Threads::number_of_non_daemon_threads()); + return _prev_conc_rs_scanning; +} + // Reuse the calculation result from init marking uint ShenandoahWorkerPolicy::calc_workers_for_final_marking() { return _prev_par_marking; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.hpp b/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.hpp index 3f47822f2200b..489be9723dd83 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.hpp @@ -31,6 +31,7 @@ class ShenandoahWorkerPolicy : AllStatic { private: static uint _prev_par_marking; static uint _prev_conc_marking; + static uint _prev_conc_rs_scanning; static uint _prev_conc_root_proc; static uint _prev_conc_refs_proc; static uint _prev_conc_evac; @@ -48,6 +49,9 @@ class ShenandoahWorkerPolicy : AllStatic { // Calculate the number of workers for concurrent marking static uint calc_workers_for_conc_marking(); + // Calculate the number of workers for remembered set scanning + static uint calc_workers_for_rs_scanning(); + // Calculate the number of workers for final marking static uint calc_workers_for_final_marking(); From 2bc25de76aa6e8434c2b0ccfe9ac8a86f21e38cb Mon Sep 17 00:00:00 2001 From: Celine Wu Date: Thu, 1 Sep 2022 19:36:47 +0000 Subject: [PATCH 147/254] Log rotation Reviewed-by: wkemper --- .../shenandoahHeapRegionCounters.cpp | 1 + .../gc/shenandoah/shenandoahLogFileOutput.cpp | 209 +++++++++++++++++- .../gc/shenandoah/shenandoahLogFileOutput.hpp | 32 +++ .../gc/shenandoah/shenandoah_globals.hpp | 10 + .../shenandoah/TestRegionSamplingLogging.java | 4 +- .../shenandoah/TestShenandoahLogRotation.java | 75 +++++++ 6 files changed, 320 insertions(+), 11 deletions(-) create mode 100644 test/hotspot/jtreg/gc/shenandoah/TestShenandoahLogRotation.java diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp index 30bb63217c040..2b4c31d54b1ad 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp @@ -77,6 +77,7 @@ ShenandoahHeapRegionCounters::ShenandoahHeapRegionCounters() : if (ShenandoahLogRegionSampling) { _log_file = new ShenandoahLogFileOutput(ShenandoahRegionSamplingFile, _timestamp->get_value()); + _log_file->set_option(ShenandoahLogFileCount, ShenandoahLogFileSize); _log_file->initialize(tty); } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahLogFileOutput.cpp b/src/hotspot/share/gc/shenandoah/shenandoahLogFileOutput.cpp index c73580d4cd1d8..67e42d5797625 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahLogFileOutput.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahLogFileOutput.cpp @@ -32,6 +32,7 @@ #include "runtime/perfData.inline.hpp" #include "utilities/globalDefinitions.hpp" #include "utilities/defaultStream.hpp" +#include "utilities/formatBuffer.hpp" #include "gc/shenandoah/shenandoahLogFileOutput.hpp" @@ -58,8 +59,88 @@ char ShenandoahLogFileOutput::_vm_start_time_str[StartTimeBufferSize]; total += result; \ } +static uint number_of_digits(uint number) { + return number < 10 ? 1 : (number < 100 ? 2 : 3); +} + +static bool is_regular_file(const char* filename) { + struct stat st; + int ret = os::stat(filename, &st); + if (ret != 0) { + return false; + } + return (st.st_mode & S_IFMT) == S_IFREG; +} + +static bool is_fifo_file(const char* filename) { + struct stat st; + int ret = os::stat(filename, &st); + if (ret != 0) { + return false; + } + return S_ISFIFO(st.st_mode); +} + +// Try to find the next number that should be used for file rotation. +// Return UINT_MAX on error. +static uint next_file_number(const char* filename, + uint number_of_digits, + uint filecount, + outputStream* errstream) { + bool found = false; + uint next_num = 0; + + // len is filename + dot + digits + null char + size_t len = strlen(filename) + number_of_digits + 2; + char* archive_name = NEW_C_HEAP_ARRAY(char, len, mtLogging); + char* oldest_name = NEW_C_HEAP_ARRAY(char, len, mtLogging); + + for (uint i = 0; i < filecount; i++) { + int ret = jio_snprintf(archive_name, len, "%s.%0*u", + filename, number_of_digits, i); + assert(ret > 0 && static_cast(ret) == len - 1, + "incorrect buffer length calculation"); + + if (os::file_exists(archive_name) && !is_regular_file(archive_name)) { + // We've encountered something that's not a regular file among the + // possible file rotation targets. Fail immediately to prevent + // problems later. + errstream->print_cr("Possible rotation target file '%s' already exists " + "but is not a regular file.", archive_name); + next_num = UINT_MAX; + break; + } + + // Stop looking if we find an unused file name + if (!os::file_exists(archive_name)) { + next_num = i; + found = true; + break; + } + + // Keep track of oldest existing log file + if (!found + || os::compare_file_modified_times(oldest_name, archive_name) > 0) { + strcpy(oldest_name, archive_name); + next_num = i; + found = true; + } + } + + FREE_C_HEAP_ARRAY(char, oldest_name); + FREE_C_HEAP_ARRAY(char, archive_name); + return next_num; +} +void ShenandoahLogFileOutput::set_option(uint file_count, size_t rotation_size) { + if (file_count < MaxRotationFileCount) { + _file_count = file_count; + } + _rotate_size = rotation_size; +} + ShenandoahLogFileOutput::ShenandoahLogFileOutput(const char* name, jlong vm_start_time) - : _name(os::strdup_check_oom(name, mtLogging)), _file_name(NULL), _stream(NULL) { + : _name(os::strdup_check_oom(name, mtLogging)), _file_name(NULL), _archive_name(NULL), _stream(NULL), _current_file(0), _file_count(DefaultFileCount), _is_default_file_count(true), _archive_name_len(0), + _rotate_size(DefaultFileSize), _current_size(0), _rotation_semaphore(1) { set_file_name_parameters(vm_start_time); _file_name = make_file_name(name, _pid_str, _vm_start_time_str); } @@ -71,6 +152,7 @@ ShenandoahLogFileOutput::~ShenandoahLogFileOutput() { _file_name, os::strerror(errno)); } } + os::free(_archive_name); os::free(_file_name); os::free(const_cast(_name)); } @@ -90,20 +172,67 @@ bool ShenandoahLogFileOutput::flush() { } void ShenandoahLogFileOutput::initialize(outputStream* errstream) { - _stream = os::fopen(_file_name, ShenandoahLogFileOutput::FileOpenMode); - if (_stream == NULL) { - errstream->print_cr("Error opening log file '%s': %s", _file_name, os::strerror(errno)); - _file_name = make_file_name("./shenandoahSnapshots_pid%p.log", _pid_str, _vm_start_time_str); + + bool file_exist = os::file_exists(_file_name); + if (file_exist && _is_default_file_count && is_fifo_file(_file_name)) { + _file_count = 0; // Prevent file rotation for fifo's such as named pipes. + } + + if (_file_count > 0) { + // compute digits with filecount - 1 since numbers will start from 0 + _file_count_max_digits = number_of_digits(_file_count - 1); + _archive_name_len = 2 + strlen(_file_name) + _file_count_max_digits; + _archive_name = NEW_C_HEAP_ARRAY(char, _archive_name_len, mtLogging); + _archive_name[0] = 0; + } + + if (_file_count > 0 && file_exist) { + if (!is_regular_file(_file_name)) { + vm_exit_during_initialization(err_msg("Unable to log to file %s with log file rotation: " + "%s is not a regular file", _file_name, _file_name)); + } + _current_file = next_file_number(_file_name, + _file_count_max_digits, + _file_count, + errstream); + if (_current_file == UINT_MAX) { + vm_exit_during_initialization("Current file reaches the maximum for integer. Unable to initialize the log output."); + } + archive(); + increment_file_count(); + } _stream = os::fopen(_file_name, ShenandoahLogFileOutput::FileOpenMode); - errstream->print_cr("Writing to default log file: %s", _file_name); - } + if (_stream == NULL) { + vm_exit_during_initialization(err_msg("Error opening log file '%s': %s", + _file_name, os::strerror(errno))); + } + if (_file_count == 0 && is_regular_file(_file_name)) { + os::ftruncate(os::get_fileno(_stream), 0); + } } +class ShenandoahRotationLocker : public StackObj { + Semaphore& _sem; + +public: + ShenandoahRotationLocker(Semaphore& sem) : _sem(sem) { + sem.wait(); + } + + ~ShenandoahRotationLocker() { + _sem.signal(); + } +}; + int ShenandoahLogFileOutput::write_snapshot(PerfLongVariable** regions, PerfLongVariable* ts, PerfLongVariable* status, size_t num_regions, size_t region_size, size_t protocol_version) { + if (_stream == NULL) { + // An error has occurred with this output, avoid writing to it. + return 0; + } int written = 0; FileLocker flocker(_stream); @@ -112,14 +241,76 @@ int ShenandoahLogFileOutput::write_snapshot(PerfLongVariable** regions, status->get_value(), num_regions, region_size, protocol_version), written); + _current_size += written; if (num_regions > 0) { WRITE_LOG_WITH_RESULT_CHECK(jio_fprintf(_stream, "%lli", regions[0]->get_value()), written); + _current_size += written; } for (uint i = 1; i < num_regions; ++i) { WRITE_LOG_WITH_RESULT_CHECK(jio_fprintf(_stream, " %lli", regions[i]->get_value()), written); + _current_size += written; + } + jio_fprintf(_stream, "\n", written); + _current_size += written; + written = flush() ? written : -1; + if (written > 0) { + _current_size += written; + + if (should_rotate()) { + rotate(); + } } - jio_fprintf(_stream, "\n"); - return flush() ? written : -1; + + return written; +} + +void ShenandoahLogFileOutput::archive() { + assert(_archive_name != NULL && _archive_name_len > 0, "Rotation must be configured before using this function."); + int ret = jio_snprintf(_archive_name, _archive_name_len, "%s.%0*u", + _file_name, _file_count_max_digits, _current_file); + assert(ret >= 0, "Buffer should always be large enough"); + + // Attempt to remove possibly existing archived log file before we rename. + // Don't care if it fails, we really only care about the rename that follows. + remove(_archive_name); + + // Rename the file from ex hotspot.log to hotspot.log.2 + if (rename(_file_name, _archive_name) == -1) { + jio_fprintf(defaultStream::error_stream(), "Could not rename log file '%s' to '%s' (%s).\n", + _file_name, _archive_name, os::strerror(errno)); + } +} + +void ShenandoahLogFileOutput::force_rotate() { + if (_file_count == 0) { + // Rotation not possible + return; + } + + ShenandoahRotationLocker lock(_rotation_semaphore); + rotate(); +} + +void ShenandoahLogFileOutput::rotate() { + if (fclose(_stream)) { + jio_fprintf(defaultStream::error_stream(), "Error closing file '%s' during log rotation (%s).\n", + _file_name, os::strerror(errno)); + } + + // Archive the current log file + archive(); + + // Open the active log file using the same stream as before + _stream = os::fopen(_file_name, FileOpenMode); + if (_stream == NULL) { + jio_fprintf(defaultStream::error_stream(), "Could not reopen file '%s' during log rotation (%s).\n", + _file_name, os::strerror(errno)); + return; + } + + // Reset accumulated size, increase current file counter, and check for file count wrap-around. + _current_size = 0; + increment_file_count(); } void ShenandoahLogFileOutput::set_file_name_parameters(jlong vm_start_time) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahLogFileOutput.hpp b/src/hotspot/share/gc/shenandoah/shenandoahLogFileOutput.hpp index 218a79494cc42..5d79513049c49 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahLogFileOutput.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahLogFileOutput.hpp @@ -28,6 +28,7 @@ #include "logging/logFileStreamOutput.hpp" #include "logging/logFileOutput.hpp" +#include "runtime/semaphore.hpp" #include "utilities/globalDefinitions.hpp" #include "runtime/perfData.inline.hpp" @@ -39,20 +40,48 @@ class ShenandoahLogFileOutput : public CHeapObj { static const char* const PidFilenamePlaceholder; static const char* const TimestampFilenamePlaceholder; static const char* const TimestampFormat; + static const size_t DefaultFileCount = 5; + static const size_t DefaultFileSize = 20 * M; static const size_t StartTimeBufferSize = 20; static const size_t PidBufferSize = 21; + static const uint MaxRotationFileCount = 1000; static char _pid_str[PidBufferSize]; static char _vm_start_time_str[StartTimeBufferSize]; const char* _name; char* _file_name; + char* _archive_name; FILE* _stream; + uint _current_file; + uint _file_count; + uint _file_count_max_digits; + bool _is_default_file_count; + + size_t _archive_name_len; + size_t _rotate_size; + size_t _current_size; + bool _write_error_is_shown; + Semaphore _rotation_semaphore; + bool parse_options(const char* options, outputStream* errstream); + void archive(); + void rotate(); char *make_file_name(const char* file_name, const char* pid_string, const char* timestamp_string); + bool should_rotate() { + return _file_count > 0 && _rotate_size > 0 && _current_size >= _rotate_size; + } + + void increment_file_count() { + _current_file++; + if (_current_file == _file_count) { + _current_file = 0; + } + } + bool flush(); public: @@ -60,6 +89,8 @@ class ShenandoahLogFileOutput : public CHeapObj { ~ShenandoahLogFileOutput(); void initialize(outputStream* errstream); + void force_rotate(); + void set_option(uint file_count, size_t rotation_size); int write_snapshot(PerfLongVariable** regions, PerfLongVariable* ts, @@ -71,6 +102,7 @@ class ShenandoahLogFileOutput : public CHeapObj { return _name; } + const char* cur_log_file_name(); static const char* const Prefix; static void set_file_name_parameters(jlong start_time); }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp index 3b0101188dc2a..802e533ba57e6 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp @@ -250,6 +250,16 @@ "to this file [default: ./shenandoahSnapshots_pid%p.log] " \ "(%p replaced with pid)") \ \ + product(uintx, ShenandoahLogFileCount, 5, "Defines the maximum number of "\ + "log files. Default is 5, maximum is 1000. Set to 0 to disable " \ + "rotation. Only includes rotated/archived files. Doesn't include "\ + "active log file.") \ + range(0, 1000) \ + \ + product(size_t, ShenandoahLogFileSize, 20 * M, "Defines the maximum size "\ + "of the log file. Files over this size will be rotated. Default " \ + "is 20MB. Set to 0 to disable rotation") \ + \ product(uintx, ShenandoahControlIntervalMin, 1, EXPERIMENTAL, \ "The minimum sleep interval for the control loop that drives " \ "the cycles. Lower values would increase GC responsiveness " \ diff --git a/test/hotspot/jtreg/gc/shenandoah/TestRegionSamplingLogging.java b/test/hotspot/jtreg/gc/shenandoah/TestRegionSamplingLogging.java index 74a2f641d298e..052b1d46430dd 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestRegionSamplingLogging.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestRegionSamplingLogging.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2017, 2018, Red Hat, Inc. All rights reserved. + * Copyright (c) 2022 Amazon.com, Inc. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -23,7 +23,7 @@ */ /* - * @test id=adaptive + * @test id=rotation * @requires vm.gc.Shenandoah * * @run main/othervm -Xmx1g -Xms1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions diff --git a/test/hotspot/jtreg/gc/shenandoah/TestShenandoahLogRotation.java b/test/hotspot/jtreg/gc/shenandoah/TestShenandoahLogRotation.java new file mode 100644 index 0000000000000..3da3933f3734e --- /dev/null +++ b/test/hotspot/jtreg/gc/shenandoah/TestShenandoahLogRotation.java @@ -0,0 +1,75 @@ + /* + * Copyright (c) 2022 Amazon.com, Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + + /* + * @test id=rotation + * @requires vm.gc.Shenandoah + * + * @run main/othervm -Xmx1g -Xms1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * -XX:+ShenandoahRegionSampling -XX:+ShenandoahRegionSampling + * -XX:+ShenandoahLogRegionSampling -XX:ShenandoahRegionSamplingFile=region-snapshots-%p.log + * -XX:ShenandoahLogFileCount=3 -XX:ShenandoahLogFileSize=100 + * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive + * TestShenandoahLogRotation + */ + + import java.io.File; + import java.util.Arrays; + import java.nio.file.Files; + + + + public class TestShenandoahLogRotation { + + static final long TARGET_MB = Long.getLong("target", 1); + + static volatile Object sink; + + public static void main(String[] args) throws Exception { + long count = TARGET_MB * 1024 * 1024 / 16; + for (long c = 0; c < count; c++) { + sink = new Object(); + Thread.sleep(1); + } + + File directory = new File("."); + File[] files = directory.listFiles((dir, name) -> name.startsWith("region-snapshots")); + System.out.println(Arrays.toString(files)); + int smallFilesNumber = 0; + for (File file : files) { + if (file.length() < 100) { + smallFilesNumber++; + } + } + // Expect one more log file since the ShenandoahLogFileCount doesn't include the active log file + int expectedNumberOfFiles = 4; + if (files.length != expectedNumberOfFiles) { + throw new Error("There are " + files.length + " logs instead of the expected " + expectedNumberOfFiles + " " + files[0].getAbsolutePath()); + } + if (smallFilesNumber > 1) { + throw new Error("There should maximum one log with size < " + 100 + "B"); + } + } + + } From 791b74dd7645198dfc9c7f98bfab1602b94bbb8b Mon Sep 17 00:00:00 2001 From: William Kemper Date: Fri, 2 Sep 2022 23:06:52 +0000 Subject: [PATCH 148/254] Fix off-by-one error when verifying object registrations Reviewed-by: kdnilsen --- .../gc/shenandoah/shenandoahCardTable.cpp | 4 ++ .../gc/shenandoah/shenandoahCardTable.hpp | 2 + .../gc/shenandoah/shenandoahGeneration.hpp | 1 + .../shenandoah/shenandoahScanRemembered.cpp | 37 ----------------- .../shenandoah/shenandoahScanRemembered.hpp | 36 ++++------------ .../shenandoahScanRemembered.inline.hpp | 41 +++++-------------- 6 files changed, 25 insertions(+), 96 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCardTable.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCardTable.cpp index 21e98f7e0b0b7..402f10a812bb9 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCardTable.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCardTable.cpp @@ -78,6 +78,10 @@ bool ShenandoahCardTable::is_dirty(MemRegion mr) { return false; } +size_t ShenandoahCardTable::last_valid_index() { + return _last_valid_index; +} + void ShenandoahCardTable::clear() { CardTable::clear(_whole_heap); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCardTable.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCardTable.hpp index b8013561e24a0..c691fac58218e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCardTable.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCardTable.hpp @@ -64,6 +64,8 @@ class ShenandoahCardTable: public CardTable { bool is_dirty(MemRegion mr); + size_t last_valid_index(); + void clear(); void clear_read_table(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index e2068883852f3..75b1d595f5a1c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -34,6 +34,7 @@ class ShenandoahHeapRegion; class ShenandoahHeapRegionClosure; class ShenandoahReferenceProcessor; +class ShenandoahHeap; class ShenandoahGeneration : public CHeapObj { private: diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp index 2b2f467478af7..837e6c8f337d3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp @@ -41,47 +41,10 @@ ShenandoahDirectCardMarkRememberedSet::ShenandoahDirectCardMarkRememberedSet(She _byte_map = _card_table->byte_for_index(0); _whole_heap_base = _card_table->addr_for(_byte_map); - _whole_heap_end = _whole_heap_base + total_card_count * CardTable::card_size(); - _byte_map_base = _byte_map - (uintptr_t(_whole_heap_base) >> _card_shift); - _overreach_map = (uint8_t *) malloc(total_card_count); - _overreach_map_base = (_overreach_map - - (uintptr_t(_whole_heap_base) >> _card_shift)); - assert(total_card_count % ShenandoahCardCluster::CardsPerCluster == 0, "Invalid card count."); assert(total_card_count > 0, "Card count cannot be zero."); - // assert(_overreach_cards != NULL); -} - -ShenandoahDirectCardMarkRememberedSet::~ShenandoahDirectCardMarkRememberedSet() { - free(_overreach_map); -} - -void ShenandoahDirectCardMarkRememberedSet::initialize_overreach(size_t first_cluster, size_t count) { - - // We can make this run faster in the future by explicitly - // unrolling the loop and doing wide writes if the compiler - // doesn't do this for us. - size_t first_card_index = first_cluster * ShenandoahCardCluster::CardsPerCluster; - uint8_t* omp = &_overreach_map[first_card_index]; - uint8_t* endp = omp + count * ShenandoahCardCluster::CardsPerCluster; - while (omp < endp) - *omp++ = CardTable::clean_card_val(); -} - -void ShenandoahDirectCardMarkRememberedSet::merge_overreach(size_t first_cluster, size_t count) { - - // We can make this run faster in the future by explicitly unrolling the loop and doing wide writes if the compiler - // doesn't do this for us. - size_t first_card_index = first_cluster * ShenandoahCardCluster::CardsPerCluster; - uint8_t* bmp = &_byte_map[first_card_index]; - uint8_t* endp = bmp + count * ShenandoahCardCluster::CardsPerCluster; - uint8_t* omp = &_overreach_map[first_card_index]; - - // dirty_card is 0, clean card is 0xff; if either *bmp or *omp is dirty, we need to mark it as dirty - while (bmp < endp) - *bmp++ &= *omp++; } ShenandoahScanRememberedTask::ShenandoahScanRememberedTask(ShenandoahObjToScanQueueSet* queue_set, diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp index aa56f9ea23658..d7657bffba3a9 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp @@ -206,12 +206,12 @@ // existing implementation. #include -#include "memory/iterator.hpp" #include "gc/shared/workerThread.hpp" #include "gc/shenandoah/shenandoahCardTable.hpp" #include "gc/shenandoah/shenandoahHeap.hpp" #include "gc/shenandoah/shenandoahHeapRegion.hpp" #include "gc/shenandoah/shenandoahTaskqueue.hpp" +#include "memory/iterator.hpp" class ShenandoahReferenceProcessor; class ShenandoahConcurrentMark; @@ -239,13 +239,8 @@ class ShenandoahDirectCardMarkRememberedSet: public CHeapObj { size_t _total_card_count; size_t _cluster_count; HeapWord *_whole_heap_base; // Points to first HeapWord of data contained within heap memory - HeapWord *_whole_heap_end; uint8_t *_byte_map; // Points to first entry within the card table uint8_t *_byte_map_base; // Points to byte_map minus the bias computed from address of heap memory - uint8_t *_overreach_map; // Points to first entry within the overreach card table - uint8_t *_overreach_map_base; // Points to overreach_map minus the bias computed from address of heap memory - - uint64_t _wide_clean_value; public: // count is the number of cards represented by the card table. @@ -253,6 +248,7 @@ class ShenandoahDirectCardMarkRememberedSet: public CHeapObj { ~ShenandoahDirectCardMarkRememberedSet(); // Card index is zero-based relative to _byte_map. + size_t last_valid_index(); size_t total_cards(); size_t card_index_for_addr(HeapWord *p); HeapWord *addr_for_card_index(size_t card_index); @@ -263,23 +259,13 @@ class ShenandoahDirectCardMarkRememberedSet: public CHeapObj { void mark_card_as_clean(size_t card_index); void mark_read_card_as_clean(size_t card_index); void mark_range_as_clean(size_t card_index, size_t num_cards); - void mark_overreach_card_as_dirty(size_t card_index); bool is_card_dirty(HeapWord *p); void mark_card_as_dirty(HeapWord *p); void mark_range_as_dirty(HeapWord *p, size_t num_heap_words); void mark_card_as_clean(HeapWord *p); void mark_range_as_clean(HeapWord *p, size_t num_heap_words); - void mark_overreach_card_as_dirty(void *p); size_t cluster_count(); - // Called by multiple GC threads at start of concurrent mark and evacuation phases. Each parallel GC thread typically - // initializes a different subranges of all overreach entries. - void initialize_overreach(size_t first_cluster, size_t count); - - // Called by GC thread at end of concurrent mark or evacuation phase. Each parallel GC thread typically merges different - // subranges of all overreach entries. - void merge_overreach(size_t first_cluster, size_t count); - // Called by GC thread at start of concurrent mark to exchange roles of read and write remembered sets. // Not currently used because mutator write barrier does not honor changes to the location of card table. void swap_remset() { _card_table->swap_card_tables(); } @@ -297,9 +283,6 @@ class ShenandoahDirectCardMarkRememberedSet: public CHeapObj { } } - HeapWord* whole_heap_base() { return _whole_heap_base; } - HeapWord* whole_heap_end() { return _whole_heap_end; } - // Instead of swap_remset, the current implementation of concurrent remembered set scanning does reset_remset // in parallel threads, each invocation processing one entire HeapRegion at a time. Processing of a region // consists of copying the write table to the read table and cleaning the write table. @@ -559,16 +542,14 @@ class ShenandoahCardCluster: public CHeapObj { _rs = rs; // TODO: We don't really need object_starts entries for every card entry. We only need these for // the card entries that correspond to old-gen memory. But for now, let's be quick and dirty. - object_starts = (crossing_info *) malloc(rs->total_cards() * sizeof(crossing_info)); - if (object_starts == nullptr) - fatal("Insufficient memory for initializing heap"); - for (size_t i = 0; i < rs->total_cards(); i++) + object_starts = NEW_C_HEAP_ARRAY(crossing_info, rs->total_cards(), mtGC); + for (size_t i = 0; i < rs->total_cards(); i++) { object_starts[i].short_word = 0; + } } ~ShenandoahCardCluster() { - if (object_starts != nullptr) - free(object_starts); + FREE_C_HEAP_ARRAY(crossing_info, object_starts); object_starts = nullptr; } @@ -892,6 +873,7 @@ class ShenandoahScanRemembered: public CHeapObj { // Card index is zero-based relative to first spanned card region. + size_t last_valid_index(); size_t total_cards(); size_t card_index_for_addr(HeapWord *p); HeapWord *addr_for_card_index(size_t card_index); @@ -902,16 +884,12 @@ class ShenandoahScanRemembered: public CHeapObj { void mark_card_as_clean(size_t card_index); void mark_read_card_as_clean(size_t card_index) { _rs->mark_read_card_clean(card_index); } void mark_range_as_clean(size_t card_index, size_t num_cards); - void mark_overreach_card_as_dirty(size_t card_index); bool is_card_dirty(HeapWord *p); void mark_card_as_dirty(HeapWord *p); void mark_range_as_dirty(HeapWord *p, size_t num_heap_words); void mark_card_as_clean(HeapWord *p); void mark_range_as_clean(HeapWord *p, size_t num_heap_words); - void mark_overreach_card_as_dirty(void *p); size_t cluster_count(); - void initialize_overreach(size_t first_cluster, size_t count); - void merge_overreach(size_t first_cluster, size_t count); // Called by GC thread at start of concurrent mark to exchange roles of read and write remembered sets. void swap_remset() { _rs->swap_remset(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp index 5beffc245931f..f92181943aa26 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp @@ -35,6 +35,11 @@ #include "gc/shenandoah/shenandoahHeapRegion.hpp" #include "gc/shenandoah/shenandoahScanRemembered.hpp" +inline size_t +ShenandoahDirectCardMarkRememberedSet::last_valid_index() { + return _card_table->last_valid_index(); +} + inline size_t ShenandoahDirectCardMarkRememberedSet::total_cards() { return _total_card_count; @@ -90,12 +95,6 @@ ShenandoahDirectCardMarkRememberedSet::mark_range_as_clean(size_t card_index, si } } -inline void -ShenandoahDirectCardMarkRememberedSet::mark_overreach_card_as_dirty(size_t card_index) { - uint8_t *bp = &_overreach_map[card_index]; - bp[0] = CardTable::dirty_card_val(); -} - inline bool ShenandoahDirectCardMarkRememberedSet::is_card_dirty(HeapWord *p) { size_t index = card_index_for_addr(p); @@ -149,12 +148,6 @@ ShenandoahDirectCardMarkRememberedSet::mark_range_as_clean(HeapWord *p, size_t n } } -inline void -ShenandoahDirectCardMarkRememberedSet::mark_overreach_card_as_dirty(void *p) { - uint8_t *bp = &_overreach_map_base[uintptr_t(p) >> _card_shift]; - bp[0] = CardTable::dirty_card_val(); -} - inline size_t ShenandoahDirectCardMarkRememberedSet::cluster_count() { return _cluster_count; @@ -271,6 +264,10 @@ ShenandoahCardCluster::get_last_start(size_t card_index) { return object_starts[card_index].offsets.last; } +template +inline size_t +ShenandoahScanRemembered::last_valid_index() { return _rs->last_valid_index(); } + template inline size_t ShenandoahScanRemembered::total_cards() { return _rs->total_cards(); } @@ -303,10 +300,6 @@ template inline void ShenandoahScanRemembered::mark_range_as_clean(size_t card_index, size_t num_cards) { _rs->mark_range_as_clean(card_index, num_cards); } -template -inline void -ShenandoahScanRemembered:: mark_overreach_card_as_dirty(size_t card_index) { _rs->mark_overreach_card_as_dirty(card_index); } - template inline bool ShenandoahScanRemembered::is_card_dirty(HeapWord *p) { return _rs->is_card_dirty(p); } @@ -327,22 +320,10 @@ template inline void ShenandoahScanRemembered:: mark_range_as_clean(HeapWord *p, size_t num_heap_words) { _rs->mark_range_as_clean(p, num_heap_words); } -template -inline void -ShenandoahScanRemembered::mark_overreach_card_as_dirty(void *p) { _rs->mark_overreach_card_as_dirty(p); } - template inline size_t ShenandoahScanRemembered::cluster_count() { return _rs->cluster_count(); } -template -inline void -ShenandoahScanRemembered::initialize_overreach(size_t first_cluster, size_t count) { _rs->initialize_overreach(first_cluster, count); } - -template -inline void -ShenandoahScanRemembered::merge_overreach(size_t first_cluster, size_t count) { _rs->merge_overreach(first_cluster, count); } - template inline void ShenandoahScanRemembered::reset_object_range(HeapWord *from, HeapWord *to) { @@ -419,9 +400,9 @@ ShenandoahScanRemembered::verify_registration(HeapWord* address, // cannot use card_index_for_addr(base_addr + offset) because it asserts arg < end of whole heap size_t end_card_index = index + offset / CardTable::card_size_in_words(); - if (end_card_index > index) { + if (end_card_index > index && end_card_index <= _rs->last_valid_index()) { // If there is a following object registered on the next card, it should begin where this object ends. - if ((base_addr + offset < _rs->whole_heap_end()) && _scc->has_object(end_card_index) && + if (_scc->has_object(end_card_index) && ((addr_for_card_index(end_card_index) + _scc->get_first_start(end_card_index)) != (base_addr + offset))) { return false; } From a3799c8d61920f58f03d0c88bd923ab53de51690 Mon Sep 17 00:00:00 2001 From: Celine Wu Date: Tue, 6 Sep 2022 17:09:15 +0000 Subject: [PATCH 149/254] Shenandoah unified logging Reviewed-by: wkemper, shade --- .../shenandoahHeapRegionCounters.cpp | 37 +- .../shenandoahHeapRegionCounters.hpp | 8 +- .../gc/shenandoah/shenandoahLogFileOutput.cpp | 408 ------------------ .../gc/shenandoah/shenandoahLogFileOutput.hpp | 110 ----- .../gc/shenandoah/shenandoah_globals.hpp | 13 - .../shenandoah/TestRegionSamplingLogging.java | 2 +- .../shenandoah/TestShenandoahLogRotation.java | 3 +- 7 files changed, 33 insertions(+), 548 deletions(-) delete mode 100644 src/hotspot/share/gc/shenandoah/shenandoahLogFileOutput.cpp delete mode 100644 src/hotspot/share/gc/shenandoah/shenandoahLogFileOutput.hpp diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp index 2b4c31d54b1ad..f0f6c6b3b6c14 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp @@ -21,7 +21,6 @@ * questions. * */ - #include "precompiled.hpp" #include "gc/shenandoah/shenandoahGeneration.hpp" @@ -29,14 +28,14 @@ #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" #include "gc/shenandoah/shenandoahHeapRegionSet.hpp" #include "gc/shenandoah/shenandoahHeapRegionCounters.hpp" -#include "gc/shenandoah/shenandoahLogFileOutput.hpp" +#include "logging/logStream.hpp" #include "memory/resourceArea.hpp" #include "runtime/atomic.hpp" #include "runtime/perfData.inline.hpp" #include "utilities/defaultStream.hpp" ShenandoahHeapRegionCounters::ShenandoahHeapRegionCounters() : - _last_sample_millis(0), _log_file(nullptr) + _last_sample_millis(0) { if (UsePerfData && ShenandoahRegionSampling) { EXCEPTION_MARK; @@ -75,17 +74,33 @@ ShenandoahHeapRegionCounters::ShenandoahHeapRegionCounters() : PerfData::U_None, CHECK); } - if (ShenandoahLogRegionSampling) { - _log_file = new ShenandoahLogFileOutput(ShenandoahRegionSamplingFile, _timestamp->get_value()); - _log_file->set_option(ShenandoahLogFileCount, ShenandoahLogFileSize); - _log_file->initialize(tty); - } } } ShenandoahHeapRegionCounters::~ShenandoahHeapRegionCounters() { if (_name_space != NULL) FREE_C_HEAP_ARRAY(char, _name_space); - if (_log_file != NULL) FREE_C_HEAP_OBJ(_log_file); +} + +void ShenandoahHeapRegionCounters::write_snapshot(PerfLongVariable** regions, + PerfLongVariable* ts, + PerfLongVariable* status, + size_t num_regions, + size_t region_size, size_t protocol_version) { + LogTarget(Debug, gc, region) lt; + if (lt.is_enabled()) { + ResourceMark rm; + LogStream ls(lt); + + ls.print_cr("%li %li %lu %lu %lu", + ts->get_value(), status->get_value(), num_regions, region_size, protocol_version); + if (num_regions > 0) { + ls.print("%li", regions[0]->get_value()); + } + for (uint i = 1; i < num_regions; ++i) { + ls.print(" %li", regions[i]->get_value()); + } + ls.cr(); + } } void ShenandoahHeapRegionCounters::update() { @@ -119,9 +134,7 @@ void ShenandoahHeapRegionCounters::update() { } // If logging enabled, dump current region snapshot to log file - if (ShenandoahLogRegionSampling && _log_file != NULL) { - _log_file->write_snapshot(_regions_data, _timestamp, _status, num_regions, rs >> 10, VERSION_NUMBER); - } + write_snapshot(_regions_data, _timestamp, _status, num_regions, rs >> 10, VERSION_NUMBER); } } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.hpp index 038ee48065f34..1376ff747d13d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.hpp @@ -27,7 +27,6 @@ #include "memory/allocation.hpp" #include "logging/logFileStreamOutput.hpp" -#include "gc/shenandoah/shenandoahLogFileOutput.hpp" /** * This provides the following in JVMStat: @@ -90,8 +89,13 @@ class ShenandoahHeapRegionCounters : public CHeapObj { PerfLongVariable* _status; volatile jlong _last_sample_millis; + void write_snapshot(PerfLongVariable** regions, + PerfLongVariable* ts, + PerfLongVariable* status, + size_t num_regions, + size_t region_size, size_t protocolVersion); + uint _count = 0; - ShenandoahLogFileOutput* _log_file; public: ShenandoahHeapRegionCounters(); ~ShenandoahHeapRegionCounters(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahLogFileOutput.cpp b/src/hotspot/share/gc/shenandoah/shenandoahLogFileOutput.cpp deleted file mode 100644 index 67e42d5797625..0000000000000 --- a/src/hotspot/share/gc/shenandoah/shenandoahLogFileOutput.cpp +++ /dev/null @@ -1,408 +0,0 @@ -/* - * Copyright (c) 2021, Amazon.com, Inc. All rights reserved. - * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#include "precompiled.hpp" -#include "jvm.h" -#include "logging/logConfiguration.hpp" -#include "logging/logFileStreamOutput.hpp" -#include "runtime/arguments.hpp" -#include "runtime/os.inline.hpp" -#include "runtime/perfData.inline.hpp" -#include "utilities/globalDefinitions.hpp" -#include "utilities/defaultStream.hpp" -#include "utilities/formatBuffer.hpp" - -#include "gc/shenandoah/shenandoahLogFileOutput.hpp" - -const char* const ShenandoahLogFileOutput::Prefix = "file="; -const char* const ShenandoahLogFileOutput::FileOpenMode = "w+"; -const char* const ShenandoahLogFileOutput::PidFilenamePlaceholder = "%p"; -const char* const ShenandoahLogFileOutput::TimestampFilenamePlaceholder = "%t"; -const char* const ShenandoahLogFileOutput::TimestampFormat = "%Y-%m-%d_%H-%M-%S"; -char ShenandoahLogFileOutput::_pid_str[PidBufferSize]; -char ShenandoahLogFileOutput::_vm_start_time_str[StartTimeBufferSize]; - -#define WRITE_LOG_WITH_RESULT_CHECK(op, total) \ -{ \ - int result = op; \ - if (result < 0) { \ - if (!_write_error_is_shown) { \ - jio_fprintf(defaultStream::error_stream(), \ - "Could not write log: %s\n", name()); \ - jio_fprintf(_stream, "\nERROR: Could not write log\n"); \ - _write_error_is_shown = true; \ - return -1; \ - } \ - } \ - total += result; \ -} - -static uint number_of_digits(uint number) { - return number < 10 ? 1 : (number < 100 ? 2 : 3); -} - -static bool is_regular_file(const char* filename) { - struct stat st; - int ret = os::stat(filename, &st); - if (ret != 0) { - return false; - } - return (st.st_mode & S_IFMT) == S_IFREG; -} - -static bool is_fifo_file(const char* filename) { - struct stat st; - int ret = os::stat(filename, &st); - if (ret != 0) { - return false; - } - return S_ISFIFO(st.st_mode); -} - -// Try to find the next number that should be used for file rotation. -// Return UINT_MAX on error. -static uint next_file_number(const char* filename, - uint number_of_digits, - uint filecount, - outputStream* errstream) { - bool found = false; - uint next_num = 0; - - // len is filename + dot + digits + null char - size_t len = strlen(filename) + number_of_digits + 2; - char* archive_name = NEW_C_HEAP_ARRAY(char, len, mtLogging); - char* oldest_name = NEW_C_HEAP_ARRAY(char, len, mtLogging); - - for (uint i = 0; i < filecount; i++) { - int ret = jio_snprintf(archive_name, len, "%s.%0*u", - filename, number_of_digits, i); - assert(ret > 0 && static_cast(ret) == len - 1, - "incorrect buffer length calculation"); - - if (os::file_exists(archive_name) && !is_regular_file(archive_name)) { - // We've encountered something that's not a regular file among the - // possible file rotation targets. Fail immediately to prevent - // problems later. - errstream->print_cr("Possible rotation target file '%s' already exists " - "but is not a regular file.", archive_name); - next_num = UINT_MAX; - break; - } - - // Stop looking if we find an unused file name - if (!os::file_exists(archive_name)) { - next_num = i; - found = true; - break; - } - - // Keep track of oldest existing log file - if (!found - || os::compare_file_modified_times(oldest_name, archive_name) > 0) { - strcpy(oldest_name, archive_name); - next_num = i; - found = true; - } - } - - FREE_C_HEAP_ARRAY(char, oldest_name); - FREE_C_HEAP_ARRAY(char, archive_name); - return next_num; -} -void ShenandoahLogFileOutput::set_option(uint file_count, size_t rotation_size) { - if (file_count < MaxRotationFileCount) { - _file_count = file_count; - } - _rotate_size = rotation_size; -} - -ShenandoahLogFileOutput::ShenandoahLogFileOutput(const char* name, jlong vm_start_time) - : _name(os::strdup_check_oom(name, mtLogging)), _file_name(NULL), _archive_name(NULL), _stream(NULL), _current_file(0), _file_count(DefaultFileCount), _is_default_file_count(true), _archive_name_len(0), - _rotate_size(DefaultFileSize), _current_size(0), _rotation_semaphore(1) { - set_file_name_parameters(vm_start_time); - _file_name = make_file_name(name, _pid_str, _vm_start_time_str); -} - -ShenandoahLogFileOutput::~ShenandoahLogFileOutput() { - if (_stream != NULL) { - if (fclose(_stream) != 0) { - jio_fprintf(defaultStream::error_stream(), "Could not close log file '%s' (%s).\n", - _file_name, os::strerror(errno)); - } - } - os::free(_archive_name); - os::free(_file_name); - os::free(const_cast(_name)); -} - -bool ShenandoahLogFileOutput::flush() { - bool result = true; - if (fflush(_stream) != 0) { - if (!_write_error_is_shown) { - jio_fprintf(defaultStream::error_stream(), - "Could not flush log: %s (%s (%d))\n", name(), os::strerror(errno), errno); - jio_fprintf(_stream, "\nERROR: Could not flush log (%d)\n", errno); - _write_error_is_shown = true; - } - result = false; - } - return result; -} - -void ShenandoahLogFileOutput::initialize(outputStream* errstream) { - - bool file_exist = os::file_exists(_file_name); - if (file_exist && _is_default_file_count && is_fifo_file(_file_name)) { - _file_count = 0; // Prevent file rotation for fifo's such as named pipes. - } - - if (_file_count > 0) { - // compute digits with filecount - 1 since numbers will start from 0 - _file_count_max_digits = number_of_digits(_file_count - 1); - _archive_name_len = 2 + strlen(_file_name) + _file_count_max_digits; - _archive_name = NEW_C_HEAP_ARRAY(char, _archive_name_len, mtLogging); - _archive_name[0] = 0; - } - - if (_file_count > 0 && file_exist) { - if (!is_regular_file(_file_name)) { - vm_exit_during_initialization(err_msg("Unable to log to file %s with log file rotation: " - "%s is not a regular file", _file_name, _file_name)); - } - _current_file = next_file_number(_file_name, - _file_count_max_digits, - _file_count, - errstream); - if (_current_file == UINT_MAX) { - vm_exit_during_initialization("Current file reaches the maximum for integer. Unable to initialize the log output."); - } - archive(); - increment_file_count(); - } - _stream = os::fopen(_file_name, ShenandoahLogFileOutput::FileOpenMode); - if (_stream == NULL) { - vm_exit_during_initialization(err_msg("Error opening log file '%s': %s", - _file_name, os::strerror(errno))); - } - if (_file_count == 0 && is_regular_file(_file_name)) { - os::ftruncate(os::get_fileno(_stream), 0); - } -} - -class ShenandoahRotationLocker : public StackObj { - Semaphore& _sem; - -public: - ShenandoahRotationLocker(Semaphore& sem) : _sem(sem) { - sem.wait(); - } - - ~ShenandoahRotationLocker() { - _sem.signal(); - } -}; - -int ShenandoahLogFileOutput::write_snapshot(PerfLongVariable** regions, - PerfLongVariable* ts, - PerfLongVariable* status, - size_t num_regions, - size_t region_size, size_t protocol_version) { - if (_stream == NULL) { - // An error has occurred with this output, avoid writing to it. - return 0; - } - int written = 0; - - FileLocker flocker(_stream); - WRITE_LOG_WITH_RESULT_CHECK(jio_fprintf(_stream, "%lli %lli %u %u %u\n", - ts->get_value(), - status->get_value(), - num_regions, - region_size, protocol_version), written); - _current_size += written; - if (num_regions > 0) { - WRITE_LOG_WITH_RESULT_CHECK(jio_fprintf(_stream, "%lli", regions[0]->get_value()), written); - _current_size += written; - } - for (uint i = 1; i < num_regions; ++i) { - WRITE_LOG_WITH_RESULT_CHECK(jio_fprintf(_stream, " %lli", regions[i]->get_value()), written); - _current_size += written; - } - jio_fprintf(_stream, "\n", written); - _current_size += written; - written = flush() ? written : -1; - if (written > 0) { - _current_size += written; - - if (should_rotate()) { - rotate(); - } - } - - return written; -} - -void ShenandoahLogFileOutput::archive() { - assert(_archive_name != NULL && _archive_name_len > 0, "Rotation must be configured before using this function."); - int ret = jio_snprintf(_archive_name, _archive_name_len, "%s.%0*u", - _file_name, _file_count_max_digits, _current_file); - assert(ret >= 0, "Buffer should always be large enough"); - - // Attempt to remove possibly existing archived log file before we rename. - // Don't care if it fails, we really only care about the rename that follows. - remove(_archive_name); - - // Rename the file from ex hotspot.log to hotspot.log.2 - if (rename(_file_name, _archive_name) == -1) { - jio_fprintf(defaultStream::error_stream(), "Could not rename log file '%s' to '%s' (%s).\n", - _file_name, _archive_name, os::strerror(errno)); - } -} - -void ShenandoahLogFileOutput::force_rotate() { - if (_file_count == 0) { - // Rotation not possible - return; - } - - ShenandoahRotationLocker lock(_rotation_semaphore); - rotate(); -} - -void ShenandoahLogFileOutput::rotate() { - if (fclose(_stream)) { - jio_fprintf(defaultStream::error_stream(), "Error closing file '%s' during log rotation (%s).\n", - _file_name, os::strerror(errno)); - } - - // Archive the current log file - archive(); - - // Open the active log file using the same stream as before - _stream = os::fopen(_file_name, FileOpenMode); - if (_stream == NULL) { - jio_fprintf(defaultStream::error_stream(), "Could not reopen file '%s' during log rotation (%s).\n", - _file_name, os::strerror(errno)); - return; - } - - // Reset accumulated size, increase current file counter, and check for file count wrap-around. - _current_size = 0; - increment_file_count(); -} - -void ShenandoahLogFileOutput::set_file_name_parameters(jlong vm_start_time) { - int res = jio_snprintf(_pid_str, sizeof(_pid_str), "%d", os::current_process_id()); - assert(res > 0, "PID buffer too small"); - - struct tm local_time; - time_t utc_time = vm_start_time / 1000; - os::localtime_pd(&utc_time, &local_time); - res = (int)strftime(_vm_start_time_str, sizeof(_vm_start_time_str), TimestampFormat, &local_time); - assert(res > 0, "VM start time buffer too small."); -} - -char* ShenandoahLogFileOutput::make_file_name(const char* file_name, - const char* pid_string, - const char* timestamp_string) { - char* result = NULL; - - // Lets start finding out if we have any %d and/or %t in the name. - // We will only replace the first occurrence of any placeholder - const char* pid = strstr(file_name, PidFilenamePlaceholder); - const char* timestamp = strstr(file_name, TimestampFilenamePlaceholder); - - if (pid == NULL && timestamp == NULL) { - // We found no place-holders, return the simple filename - return os::strdup_check_oom(file_name, mtLogging); - } - - // At least one of the place-holders were found in the file_name - const char* first = ""; - size_t first_pos = SIZE_MAX; - size_t first_replace_len = 0; - - const char* second = ""; - size_t second_pos = SIZE_MAX; - size_t second_replace_len = 0; - - // If we found a %p, then setup our variables accordingly - if (pid != NULL) { - if (timestamp == NULL || pid < timestamp) { - first = pid_string; - first_pos = pid - file_name; - first_replace_len = strlen(PidFilenamePlaceholder); - } else { - second = pid_string; - second_pos = pid - file_name; - second_replace_len = strlen(PidFilenamePlaceholder); - } - } - - if (timestamp != NULL) { - if (pid == NULL || timestamp < pid) { - first = timestamp_string; - first_pos = timestamp - file_name; - first_replace_len = strlen(TimestampFilenamePlaceholder); - } else { - second = timestamp_string; - second_pos = timestamp - file_name; - second_replace_len = strlen(TimestampFilenamePlaceholder); - } - } - - size_t first_len = strlen(first); - size_t second_len = strlen(second); - - // Allocate the new buffer, size it to hold all we want to put in there +1. - size_t result_len = strlen(file_name) + first_len - first_replace_len + second_len - second_replace_len; - result = NEW_C_HEAP_ARRAY(char, result_len + 1, mtLogging); - - // Assemble the strings - size_t file_name_pos = 0; - size_t i = 0; - while (i < result_len) { - if (file_name_pos == first_pos) { - // We are in the range of the first placeholder - strcpy(result + i, first); - // Bump output buffer position with length of replacing string - i += first_len; - // Bump source buffer position to skip placeholder - file_name_pos += first_replace_len; - } else if (file_name_pos == second_pos) { - // We are in the range of the second placeholder - strcpy(result + i, second); - i += second_len; - file_name_pos += second_replace_len; - } else { - // Else, copy char by char of the original file - result[i] = file_name[file_name_pos++]; - i++; - } - } - // Add terminating char - result[result_len] = '\0'; - return result; -} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahLogFileOutput.hpp b/src/hotspot/share/gc/shenandoah/shenandoahLogFileOutput.hpp deleted file mode 100644 index 5d79513049c49..0000000000000 --- a/src/hotspot/share/gc/shenandoah/shenandoahLogFileOutput.hpp +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright (c) 2021, Amazon.com, Inc. All rights reserved. - * Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHLOGFILEOUTPUT_HPP -#define SHARE_GC_SHENANDOAH_SHENANDOAHLOGFILEOUTPUT_HPP - -#include "logging/logFileStreamOutput.hpp" -#include "logging/logFileOutput.hpp" -#include "runtime/semaphore.hpp" -#include "utilities/globalDefinitions.hpp" -#include "runtime/perfData.inline.hpp" - -// Log file output to capture Shenandoah GC data. - -class ShenandoahLogFileOutput : public CHeapObj { -private: - static const char* const FileOpenMode; - static const char* const PidFilenamePlaceholder; - static const char* const TimestampFilenamePlaceholder; - static const char* const TimestampFormat; - static const size_t DefaultFileCount = 5; - static const size_t DefaultFileSize = 20 * M; - static const size_t StartTimeBufferSize = 20; - static const size_t PidBufferSize = 21; - static const uint MaxRotationFileCount = 1000; - static char _pid_str[PidBufferSize]; - static char _vm_start_time_str[StartTimeBufferSize]; - - const char* _name; - char* _file_name; - char* _archive_name; - FILE* _stream; - - uint _current_file; - uint _file_count; - uint _file_count_max_digits; - bool _is_default_file_count; - - size_t _archive_name_len; - size_t _rotate_size; - size_t _current_size; - - bool _write_error_is_shown; - - Semaphore _rotation_semaphore; - - bool parse_options(const char* options, outputStream* errstream); - void archive(); - void rotate(); - char *make_file_name(const char* file_name, const char* pid_string, const char* timestamp_string); - - bool should_rotate() { - return _file_count > 0 && _rotate_size > 0 && _current_size >= _rotate_size; - } - - void increment_file_count() { - _current_file++; - if (_current_file == _file_count) { - _current_file = 0; - } - } - - bool flush(); - -public: - ShenandoahLogFileOutput(const char *name, jlong vm_start_time); - ~ShenandoahLogFileOutput(); - - void initialize(outputStream* errstream); - void force_rotate(); - void set_option(uint file_count, size_t rotation_size); - - int write_snapshot(PerfLongVariable** regions, - PerfLongVariable* ts, - PerfLongVariable* status, - size_t num_regions, - size_t region_size, size_t protocolVersion); - - const char* name() const { - return _name; - } - - const char* cur_log_file_name(); - static const char* const Prefix; - static void set_file_name_parameters(jlong start_time); -}; -#endif //SHARE_GC_SHENANDOAH_SHENANDOAHLOGFILEOUTPUT_HPP - diff --git a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp index 802e533ba57e6..883cff5dde99e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp @@ -241,25 +241,12 @@ "the samples. Higher values provide more fidelity, at expense " \ "of more sampling overhead.") \ \ - product(bool, ShenandoahLogRegionSampling, false, \ - "Save region sampling stream to ShenandoahRegionSamplingFile") \ - \ product(ccstr, ShenandoahRegionSamplingFile, \ "./shenandoahSnapshots_pid%p.log", \ "If ShenandoahLogRegionSampling is on, save sampling data stream "\ "to this file [default: ./shenandoahSnapshots_pid%p.log] " \ "(%p replaced with pid)") \ \ - product(uintx, ShenandoahLogFileCount, 5, "Defines the maximum number of "\ - "log files. Default is 5, maximum is 1000. Set to 0 to disable " \ - "rotation. Only includes rotated/archived files. Doesn't include "\ - "active log file.") \ - range(0, 1000) \ - \ - product(size_t, ShenandoahLogFileSize, 20 * M, "Defines the maximum size "\ - "of the log file. Files over this size will be rotated. Default " \ - "is 20MB. Set to 0 to disable rotation") \ - \ product(uintx, ShenandoahControlIntervalMin, 1, EXPERIMENTAL, \ "The minimum sleep interval for the control loop that drives " \ "the cycles. Lower values would increase GC responsiveness " \ diff --git a/test/hotspot/jtreg/gc/shenandoah/TestRegionSamplingLogging.java b/test/hotspot/jtreg/gc/shenandoah/TestRegionSamplingLogging.java index 052b1d46430dd..c985bbc310dd0 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestRegionSamplingLogging.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestRegionSamplingLogging.java @@ -28,7 +28,7 @@ * * @run main/othervm -Xmx1g -Xms1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions * -XX:+ShenandoahRegionSampling -XX:+ShenandoahRegionSampling - * -XX:+ShenandoahLogRegionSampling -XX:ShenandoahRegionSamplingFile=region-snapshots-%p.log + * -Xlog:gc+region=debug:region-snapshots-%p.log::filesize=100,filecount=3 * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive * TestRegionSamplingLogging */ diff --git a/test/hotspot/jtreg/gc/shenandoah/TestShenandoahLogRotation.java b/test/hotspot/jtreg/gc/shenandoah/TestShenandoahLogRotation.java index 3da3933f3734e..54ce716d785e8 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestShenandoahLogRotation.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestShenandoahLogRotation.java @@ -28,8 +28,7 @@ * * @run main/othervm -Xmx1g -Xms1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions * -XX:+ShenandoahRegionSampling -XX:+ShenandoahRegionSampling - * -XX:+ShenandoahLogRegionSampling -XX:ShenandoahRegionSamplingFile=region-snapshots-%p.log - * -XX:ShenandoahLogFileCount=3 -XX:ShenandoahLogFileSize=100 + * -Xlog:gc+region=debug:region-snapshots-%p.log::filesize=100,filecount=3 * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive * TestShenandoahLogRotation */ From 22ff4b919f7b8af0fd880f56a3a23edd701bb322 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Tue, 1 Nov 2022 16:28:39 +0000 Subject: [PATCH 150/254] Fix assertion error with advance promotion budgeting Reviewed-by: rkennke --- .../gc/shenandoah/shenandoahGeneration.cpp | 332 ++++++++++++------ .../gc/shenandoah/shenandoahGeneration.hpp | 6 +- 2 files changed, 229 insertions(+), 109 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index 19c63af676128..ac513203999f4 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -215,12 +215,11 @@ void ShenandoahGeneration::prepare_gc() { void ShenandoahGeneration::compute_evacuation_budgets(ShenandoahHeap* heap, bool* preselected_regions, ShenandoahCollectionSet* collection_set, - size_t &old_regions_loaned_for_young_evac, size_t ®ions_available_to_loan, - size_t &minimum_evacuation_reserve, size_t &consumed_by_advance_promotion) { + size_t &consumed_by_advance_promotion) { size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); - minimum_evacuation_reserve = ShenandoahOldCompactionReserve * region_size_bytes; - old_regions_loaned_for_young_evac = 0; - regions_available_to_loan = 0; + size_t regions_available_to_loan = 0; + size_t minimum_evacuation_reserve = ShenandoahOldCompactionReserve * region_size_bytes; + size_t old_regions_loaned_for_young_evac = 0; consumed_by_advance_promotion = 0; if (heap->mode()->is_generational()) { ShenandoahGeneration* old_generation = heap->old_generation(); @@ -270,12 +269,6 @@ void ShenandoahGeneration::compute_evacuation_budgets(ShenandoahHeap* heap, bool // there can be slight discrepancies here. minimum_evacuation_reserve = old_generation->available(); } - if (old_evacuation_reserve < minimum_evacuation_reserve) { - // Even if there's nothing to be evacuated on this cycle, we still need to reserve this memory for future - // evacuations. It is ok to loan this memory to young-gen if we don't need it for evacuation on this pass. - avail_evac_reserve_for_loan_to_young_gen = minimum_evacuation_reserve - old_evacuation_reserve; - old_evacuation_reserve = minimum_evacuation_reserve; - } heap->set_old_evac_reserve(old_evacuation_reserve); heap->reset_old_evac_expended(); @@ -361,12 +354,21 @@ void ShenandoahGeneration::compute_evacuation_budgets(ShenandoahHeap* heap, bool // old evacuation can pack into existing partially used regions. young evacuation and loans for young allocations // need to target regions that do not already hold any old-gen objects. Round down. regions_available_to_loan = old_generation->free_unaffiliated_regions(); - consumed_by_advance_promotion = _heuristics->select_aged_regions(old_generation->available() - old_evacuation_reserve, - num_regions, preselected_regions); + + size_t required_evacuation_reserve; + // Memory evacuated from old-gen on this pass will be available to hold old-gen evacuations in next pass. + if (old_evacuation_reserve > minimum_evacuation_reserve) { + required_evacuation_reserve = 0; + } else { + required_evacuation_reserve = minimum_evacuation_reserve - old_evacuation_reserve; + } + + consumed_by_advance_promotion = _heuristics->select_aged_regions( + old_generation->available() - old_evacuation_reserve - required_evacuation_reserve, num_regions, preselected_regions); size_t net_available_old_regions = (old_generation->available() - old_evacuation_reserve - consumed_by_advance_promotion) / region_size_bytes; - if (regions_available_to_loan > net_available_old_regions) { + if (regions_available_to_loan > net_available_old_regions) { regions_available_to_loan = net_available_old_regions; } @@ -385,8 +387,6 @@ void ShenandoahGeneration::compute_evacuation_budgets(ShenandoahHeap* heap, bool // In this case, there's no memory available for new allocations while evacuating and updating, unless we // find more old-gen memory to borrow below. } - } else { - old_regions_loaned_for_young_evac = 0; } // In generational mode, we may end up choosing a young collection set that contains so many promotable objects // that there is not sufficient space in old generation to hold the promoted objects. That is ok because we have @@ -407,10 +407,26 @@ void ShenandoahGeneration::compute_evacuation_budgets(ShenandoahHeap* heap, bool // Having chosen the collection set, adjust the budgets for generatioal mode based on its composition. Note // that young_generation->available() now knows about recently discovered immediate garbage. + void ShenandoahGeneration::adjust_evacuation_budgets(ShenandoahHeap* heap, ShenandoahCollectionSet* collection_set, - size_t old_regions_loaned_for_young_evac, size_t regions_available_to_loan, - size_t minimum_evacuation_reserve, size_t consumed_by_advance_promotion) { + size_t consumed_by_advance_promotion) { + // We may find that old_evacuation_reserve and/or loaned_for_young_evacuation are not fully consumed, in which case we may + // be able to increase regions_available_to_loan + + // The role of adjust_evacuation_budgets() is to compute the correct value of regions_available_to_loan and to make + // effective use of this memory, including the remnant memory within these regions that may result from rounding loan to + // integral number of regions. Excess memory that is available to be loaned is applied to an allocation supplement, + // which allows mutators to allocate memory beyond the current capacity of young-gen on the promise that the loan + // will be repaid as soon as we finish updating references for the recently evacuated collection set. + + // We cannot recalculate regions_available_to_loan by simply dividing old_generation->available() by region_size_bytes + // because the available memory may be distributed between many partially occupied regions that are already holding old-gen + // objects. Memory in partially occupied regions is not "available" to be loaned. Note that an increase in old-gen + // available that results from a decrease in memory consumed by old evacuation is not necessarily available to be loaned + // to young-gen. + if (heap->mode()->is_generational()) { + size_t old_regions_loaned_for_young_evac, regions_available_to_loan; size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); ShenandoahOldGeneration* old_generation = heap->old_generation(); ShenandoahYoungGeneration* young_generation = heap->young_generation(); @@ -421,9 +437,14 @@ void ShenandoahGeneration::adjust_evacuation_budgets(ShenandoahHeap* heap, Shena size_t immediate_garbage = collection_set->get_immediate_trash(); size_t old_available = old_generation->available(); size_t young_available = young_generation->available() + immediate_garbage; + size_t loaned_regions = 0; + size_t available_loan_remnant = 0; // loaned memory that is not yet dedicated to any particular budget - assert(consumed_by_advance_promotion >= collection_set->get_young_bytes_to_be_promoted() * ShenandoahEvacWaste, - "Advance promotion should be at least young_bytes_to_be_promoted * ShenandoahEvacWaste"); + assert(((consumed_by_advance_promotion * 33) / 32) >= collection_set->get_young_bytes_to_be_promoted() * ShenandoahEvacWaste, + "Advance promotion (" SIZE_FORMAT ") should be at least young_bytes_to_be_promoted (" SIZE_FORMAT + ")* ShenandoahEvacWaste, totalling: " SIZE_FORMAT ", within round-off errors of up to 3.125%%", + consumed_by_advance_promotion, collection_set->get_young_bytes_to_be_promoted(), + (size_t) (collection_set->get_young_bytes_to_be_promoted() * ShenandoahEvacWaste)); assert(consumed_by_advance_promotion <= (collection_set->get_young_bytes_to_be_promoted() * ShenandoahEvacWaste * 33) / 32, "Round-off errors should be less than 3.125%%, consumed by advance: " SIZE_FORMAT ", promoted: " SIZE_FORMAT, @@ -439,11 +460,6 @@ void ShenandoahGeneration::adjust_evacuation_budgets(ShenandoahHeap* heap, Shena old_evacuated_committed = old_evacuation_reserve; } else if (old_evacuated_committed < old_evacuation_reserve) { // This may happen if the old-gen collection consumes less than full budget. - - // If we shrink old_evacuation_reserve by more than a region size, we can expand regions_available_to_loan. - // Can only give back regions that are fully unused, so round down. - size_t old_evac_regions_unused = (old_evacuation_reserve - old_evacuated_committed) / region_size_bytes; - regions_available_to_loan += old_evac_regions_unused; old_evacuation_reserve = old_evacuated_committed; heap->set_old_evac_reserve(old_evacuation_reserve); } @@ -456,7 +472,7 @@ void ShenandoahGeneration::adjust_evacuation_budgets(ShenandoahHeap* heap, Shena size_t young_evacuated = collection_set->get_young_bytes_reserved_for_evacuation() - young_promoted; size_t young_evacuated_reserve_used = (size_t) (ShenandoahEvacWaste * young_evacuated); - heap->set_young_evac_reserve(young_evacuated_reserve_used); + // We'll invoke heap->set_young_evac_reserve() further below, after we make additional adjustments to its value // Adjust old_regions_loaned_for_young_evac to feed into calculations of promoted_reserve if (young_evacuated_reserve_used > young_available) { @@ -465,61 +481,173 @@ void ShenandoahGeneration::adjust_evacuation_budgets(ShenandoahHeap* heap, Shena // region_size_bytes is a power of 2. loan an integral number of regions. size_t revised_loan_for_young_evacuation = (short_fall + region_size_bytes - 1) / region_size_bytes; - // Undo the previous loan - regions_available_to_loan += old_regions_loaned_for_young_evac; - old_regions_loaned_for_young_evac = revised_loan_for_young_evacuation; + // available_loan_remnant represents memory loaned from old-gen but not required for young evacuation. + // This is the excess loaned memory that results from rounding the required loan up to an integral number + // of heap regions. This will be dedicated to alloc_supplement below. + available_loan_remnant = (revised_loan_for_young_evacuation * region_size_bytes) - short_fall; - // And make a new loan - assert(regions_available_to_loan > old_regions_loaned_for_young_evac, "Cannot loan regions that we do not have"); - regions_available_to_loan -= revised_loan_for_young_evacuation; + // We previously loaned more than was required by young-gen evacuation. So claw some of this memory back. + old_regions_loaned_for_young_evac = revised_loan_for_young_evacuation; + loaned_regions = old_regions_loaned_for_young_evac; } else { - // Undo the prevous loan - regions_available_to_loan += old_regions_loaned_for_young_evac; + // Undo the prevous loan, if any. old_regions_loaned_for_young_evac = 0; + loaned_regions = 0; } - size_t old_bytes_loaned_for_young_evac = old_regions_loaned_for_young_evac * region_size_bytes; - size_t old_bytes_reserved_for_alloc_supplement = 0; - size_t old_regions_reserved_for_alloc_supplement = 0; - // Need to enforce that old_evacuated_committed + old_bytes_loaned_for_young_evac >= minimum_evacuation_reserve + size_t old_bytes_loaned_for_young_evac = old_regions_loaned_for_young_evac * region_size_bytes - available_loan_remnant; + + // Recompute regions_available_to_loan based on possible changes to old_regions_loaned_for_young_evac and + // old_evacuation_reserve. + + // Any decrease in old_regions_loaned_for_young_evac are immediately available to be loaned + // However, a change to old_evacuation_reserve() is not necessarily available to loan, because this memory may + // reside within many fragments scattered throughout old-gen. + + regions_available_to_loan = old_generation->free_unaffiliated_regions(); + size_t working_old_available = old_generation->available(); + + assert(regions_available_to_loan * region_size_bytes <= working_old_available, + "Regions available to loan must be less than available memory"); + + // fragmented_old_total is the amount of memory in old-gen beyond regions_available_to_loan that is otherwise not + // yet dedicated to a particular budget. This memory can be used for promotion_reserve. + size_t fragmented_old_total = working_old_available - regions_available_to_loan * region_size_bytes; + + // fragmented_old_usage is the memory that is dedicated to holding evacuated old-gen objects, which does not need + // to be an integral number of regions. + size_t fragmented_old_usage = old_evacuated_committed + consumed_by_advance_promotion; + + + + if (fragmented_old_total >= fragmented_old_usage) { + // Seems this will be rare. In this case, all of the memory required for old-gen evacuations and promotions can be + // taken from the existing fragments within old-gen. Reduce this fragmented total by this amount. + fragmented_old_total -= fragmented_old_usage; + // And reduce regions_available_to_loan by the regions dedicated to young_evac. + regions_available_to_loan -= old_regions_loaned_for_young_evac; + } else { + // In this case, we need to dedicate some of the regions_available_to_loan to hold the results of old-gen evacuations + // and promotions. + + size_t unaffiliated_memory_required_for_old = fragmented_old_usage - fragmented_old_total; + size_t unaffiliated_regions_used_by_old = (unaffiliated_memory_required_for_old + region_size_bytes - 1) / region_size_bytes; + regions_available_to_loan -= (unaffiliated_regions_used_by_old + old_regions_loaned_for_young_evac); + + size_t memory_for_promotions_and_old_evac = fragmented_old_total + unaffiliated_regions_used_by_old; + size_t memory_required_for_promotions_and_old_evac = fragmented_old_usage; + size_t excess_fragmented = memory_for_promotions_and_old_evac - memory_required_for_promotions_and_old_evac; + fragmented_old_total = excess_fragmented; + } + + // Subtract from working_old_available old_evacuated_committed and consumed_by_advance_promotion + working_old_available -= fragmented_old_usage; + // And also subtract out the regions loaned for young evacuation + working_old_available -= old_regions_loaned_for_young_evac * region_size_bytes; + + // Assure that old_evacuated_committed + old_bytes_loaned_for_young_evac >= the minimum evacuation reserve // in order to prevent promotion reserve from violating minimum evacuation reserve. - if (old_evacuated_committed + old_bytes_loaned_for_young_evac < minimum_evacuation_reserve) { - // Reserve some of the regions available to loan for use as allocation supplement to assure memory not consumed by promotion - size_t excess_bytes = minimum_evacuation_reserve - (old_evacuated_committed + old_bytes_loaned_for_young_evac); - size_t excess_regions = (excess_bytes - 1 + region_size_bytes) / region_size_bytes; - if (regions_available_to_loan <= excess_regions) { - excess_regions = regions_available_to_loan; - // Since we can't reserve entire excess for alloc supplement, pretend more is consumed by old-evacuation - old_evacuated_committed = - minimum_evacuation_reserve - old_bytes_loaned_for_young_evac - excess_regions * region_size_bytes; + size_t old_regions_reserved_for_alloc_supplement = 0; + size_t old_bytes_reserved_for_alloc_supplement = 0; + size_t reserved_bytes_for_future_old_evac = 0; + + old_bytes_reserved_for_alloc_supplement = available_loan_remnant; + available_loan_remnant = 0; + + // Memory that has been loaned for young evacuations and old-gen regions in the current mixed-evacuation collection + // set will be available to hold future old-gen evacuations. If this memory is less than the desired amount of memory + // set aside for old-gen compaction reserve, try to set aside additional memory so that it will be available during + // the next mixed evacuation cycle. Note that memory loaned to young-gen for allocation supplement is excluded from + // the old-gen promotion reserve. + size_t future_evac_reserve_regions = old_regions_loaned_for_young_evac + collection_set->get_old_region_count(); + size_t collected_regions = collection_set->get_young_region_count(); + + if (future_evac_reserve_regions < ShenandoahOldCompactionReserve) { + // Require that we loan more memory for holding young evacuations to assure that we have adequate reserves to receive + // old-gen evacuations during subsequent collections. Loaning this memory for an allocation supplement does not + // satisfy our needs because newly allocated objects are not necessarily counter-balanced by reclaimed collection + // set regions. + + // Put this memory into reserve by identifying it as old_regions_loaned_for_young_evac + size_t additional_regions_to_loan = ShenandoahOldCompactionReserve - future_evac_reserve_regions; + + // We can loan additional regions to be repaid from the anticipated recycling of young collection set regions + // provided that these regions are currently available within old-gen memory. + size_t collected_regions_to_loan; + if (collected_regions >= additional_regions_to_loan) { + collected_regions_to_loan = additional_regions_to_loan; + additional_regions_to_loan = 0; + } else if (collected_regions > 0) { + collected_regions_to_loan = collected_regions; + additional_regions_to_loan -= collected_regions_to_loan; + } else { + collected_regions_to_loan = 0; + } + + if (collected_regions_to_loan > 0) { + // We're evacuating at least this many regions, it's ok to use these regions for allocation supplement since + // we'll be able to repay the loan at end of this GC pass, assuming the regions are available. + if (collected_regions_to_loan > regions_available_to_loan) { + collected_regions_to_loan = regions_available_to_loan; + } + old_bytes_reserved_for_alloc_supplement += collected_regions_to_loan * region_size_bytes; + regions_available_to_loan -= collected_regions_to_loan; + loaned_regions += collected_regions_to_loan; + working_old_available -= collected_regions_to_loan * region_size_bytes; + } + + // If there's still memory that we want to exclude from the current promotion reserve, but we are unable to loan + // this memory because fully empty old-gen regions are not available, decrement the working_old_available to make + // sure that this memory is not used to hold the results of old-gen evacuation. + if (additional_regions_to_loan > regions_available_to_loan) { + size_t unloaned_regions = additional_regions_to_loan - regions_available_to_loan; + size_t unloaned_bytes = unloaned_regions * region_size_bytes; + + if (working_old_available < unloaned_bytes) { + // We're in dire straits. We won't be able to reserve all the memory that we want to make available for the + // next old-gen evacuation. We'll reserve as much of it as possible. Setting working_old_available to zero + // means there will be no promotion except for the advance promotion. Note that if some advance promotion fails, + // the object will be evacuated to young-gen so we should still end up reclaiming the entire advance promotion + // collection set. + reserved_bytes_for_future_old_evac = working_old_available; + working_old_available = 0; + } else { + reserved_bytes_for_future_old_evac = unloaned_bytes; + working_old_available -= unloaned_bytes; + } + size_t regions_reserved_for_future_old_evac = + (reserved_bytes_for_future_old_evac + region_size_bytes - 1) / region_size_bytes; + + if (regions_reserved_for_future_old_evac < regions_available_to_loan) { + regions_available_to_loan -= regions_reserved_for_future_old_evac; + } else { + regions_available_to_loan = 0; + } + + // Since we're in dire straits, zero out fragmented_old_total so this won't be used for promotion; + if (working_old_available > fragmented_old_total) { + working_old_available -= fragmented_old_total; + } else { + working_old_available = 0; + } + fragmented_old_total = 0; } - regions_available_to_loan -= excess_regions; - old_bytes_reserved_for_alloc_supplement = excess_regions * region_size_bytes; - old_regions_reserved_for_alloc_supplement = excess_regions; } + // Establish young_evac_reserve so that this young-gen memory is not used for new allocations, allowing the memory + // to be returned to old-gen as soon as the current collection set regions are reclaimed. + heap->set_young_evac_reserve(young_evacuated_reserve_used); + // Limit promoted_reserve so that we can set aside memory to be loaned from old-gen to young-gen. This // value is not "critical". If we underestimate, certain promotions will simply be deferred. If we put // "all the rest" of old-gen memory into the promotion reserve, we'll have nothing left to loan to young-gen // during the evac and update phases of GC. So we "limit" the sizes of the promotion budget to be the smaller of: // - // 1. old_gen->available - (old_evacuation_committed + old_bytes_loaned_for_young_evac + consumed_by_advance_promotion - // + old_bytes_[already]_ reserved_for_alloc_supplement) + // 1. old_available + // (old_available is old_gen->available() - + // (old_evacuated_committed + consumed_by_advance_promotion + loaned_for_young_evac + reserved_for_alloc_supplement)) // 2. young bytes reserved for evacuation (we can't promote more than young is evacuating) - - assert(old_available > old_evacuated_committed, "Cannot evacuate more than available"); - assert(old_available > old_evacuated_committed + old_bytes_loaned_for_young_evac, - "Cannot loan young evac more than available"); - assert(old_available > old_evacuated_committed + old_bytes_loaned_for_young_evac + consumed_by_advance_promotion, - "Cannot promote more than available"); - assert(old_available > (old_evacuated_committed + old_bytes_loaned_for_young_evac + - consumed_by_advance_promotion + old_bytes_reserved_for_alloc_supplement), - "Cannot loan for alloc supplement more than available"); - - size_t promotion_reserve = regions_available_to_loan * region_size_bytes; - assert(promotion_reserve <= old_available - (old_evacuated_committed + consumed_by_advance_promotion + - old_bytes_loaned_for_young_evac + old_bytes_reserved_for_alloc_supplement), - "Byte reserves do not match region reserves"); + size_t promotion_reserve = working_old_available; // We experimented with constraining promoted_reserve to be no larger than 4 times the size of previously_promoted, // but this constraint was too limiting, resulting in failure of legitimate promotions. This was tried before we @@ -542,22 +670,27 @@ void ShenandoahGeneration::adjust_evacuation_budgets(ShenandoahHeap* heap, Shena // young_evacuation_reserve_used does not include live memory within tenure-aged regions. promotion_reserve = young_evacuated_reserve_used; } + assert(working_old_available >= promotion_reserve, "Cannot reserve for promotion more than is available"); + working_old_available -= promotion_reserve; + // Having reserved this memory for promotion, the regions are no longer available to be loaned. + size_t regions_consumed_by_promotion_reserve = (promotion_reserve + region_size_bytes - 1) / region_size_bytes; + if (regions_consumed_by_promotion_reserve > regions_available_to_loan) { + // This can happen if the promotion reserve makes use of memory that is fragmented between many partially available + // old-gen regions. + regions_available_to_loan = 0; + } else { + regions_available_to_loan -= regions_consumed_by_promotion_reserve; + } - assert(old_available >= (promotion_reserve + old_evacuated_committed + old_bytes_loaned_for_young_evac + - consumed_by_advance_promotion + old_bytes_reserved_for_alloc_supplement), - "Budget exceeds available old-gen memory"); - log_debug(gc)("Old available: " SIZE_FORMAT ", Original promotion reserve: " SIZE_FORMAT ", Old evacuation reserve: " - SIZE_FORMAT ", Advance promotion reserve supplement: " SIZE_FORMAT - ", Old loaned for young evacuation: " SIZE_FORMAT ", Old reserved for alloc supplement: " SIZE_FORMAT, - old_available, promotion_reserve, old_evacuated_committed, consumed_by_advance_promotion, + log_debug(gc)("old_gen->available(): " SIZE_FORMAT " divided between promotion reserve: " SIZE_FORMAT + ", old evacuation reserve: " SIZE_FORMAT ", advance promotion reserve supplement: " SIZE_FORMAT + ", old loaned for young evacuation: " SIZE_FORMAT ", old reserved for alloc supplement: " SIZE_FORMAT, + old_generation->available(), promotion_reserve, old_evacuated_committed, consumed_by_advance_promotion, old_regions_loaned_for_young_evac * region_size_bytes, old_bytes_reserved_for_alloc_supplement); promotion_reserve += consumed_by_advance_promotion; heap->set_promoted_reserve(promotion_reserve); - size_t promotion_regions = (promotion_reserve + region_size_bytes - 1) / region_size_bytes; - assert(regions_available_to_loan >= promotion_regions, "Promoting more regions than memory is available"); - regions_available_to_loan -= promotion_regions; heap->reset_promoted_expended(); if (collection_set->get_old_bytes_reserved_for_evacuation() == 0) { // Setting old evacuation reserve to zero denotes that there is no old-gen evacuation in this pass. @@ -567,15 +700,15 @@ void ShenandoahGeneration::adjust_evacuation_budgets(ShenandoahHeap* heap, Shena size_t old_gen_usage_base = old_generation->used() - collection_set->get_old_garbage(); heap->capture_old_usage(old_gen_usage_base); - // Compute the evacuation supplement, which is extra memory borrowed from old-gen that can be allocated + // Compute additional evacuation supplement, which is extra memory borrowed from old-gen that can be allocated // by mutators while GC is working on evacuation and update-refs. This memory can be temporarily borrowed // from old-gen allotment, then repaid at the end of update-refs from the recycled collection set. After // we have computed the collection set based on the parameters established above, we can make additional // loans based on our knowledge of the collection set to determine how much allocation we can allow - // during the evacuation and update-refs phases of execution. The total available supplement is the smaller of: + // during the evacuation and update-refs phases of execution. The total available supplement is the result + // of adding old_bytes_reserved_for_alloc_supplement to the smaller of: // - // 1. old_gen->available() - - // (promotion_reserve + old_evacuation_commitment + old_bytes_loaned_for_young_evac) + // 1. regions_available_to_loan * region_size_bytes // 2. The replenishment budget (number of regions in collection set - the number of regions already // under lien for the young_evacuation_reserve) // @@ -583,25 +716,19 @@ void ShenandoahGeneration::adjust_evacuation_budgets(ShenandoahHeap* heap, Shena // Regardless of how many regions may be available to be loaned, we can loan no more regions than // the total number of young regions to be evacuated. Call this the regions_for_runway. - size_t young_regions_evacuated = collection_set->get_young_region_count(); - size_t regions_for_runway = 0; - size_t already_loaned_regions = old_regions_loaned_for_young_evac + old_regions_reserved_for_alloc_supplement; - if (already_loaned_regions == 0) { - regions_for_runway = young_regions_evacuated; - } else if (young_regions_evacuated > already_loaned_regions) { - regions_for_runway = young_regions_evacuated - already_loaned_regions; - } else { - regions_for_runway = 0; - } + if (regions_available_to_loan > 0 && (collected_regions > loaned_regions)) { + assert(regions_available_to_loan * region_size_bytes <= working_old_available, + "regions_available_to_loan should not exceed working_old_available"); - if (regions_available_to_loan > regions_for_runway) { - regions_available_to_loan -= regions_for_runway; - } else { - regions_for_runway = regions_available_to_loan; - regions_available_to_loan = 0; + size_t additional_regions_to_loan = collected_regions - loaned_regions; + if (additional_regions_to_loan > regions_available_to_loan) { + additional_regions_to_loan = regions_available_to_loan; + } + loaned_regions += additional_regions_to_loan; + old_bytes_reserved_for_alloc_supplement += additional_regions_to_loan * region_size_bytes; + working_old_available -= additional_regions_to_loan * region_size_bytes; } - - size_t allocation_supplement = regions_for_runway * region_size_bytes + old_bytes_reserved_for_alloc_supplement; + size_t allocation_supplement = old_bytes_reserved_for_alloc_supplement; heap->set_alloc_supplement_reserve(allocation_supplement); // TODO: young_available, which feeds into alloc_budget_evac_and_update is lacking memory available within @@ -621,9 +748,6 @@ void ShenandoahGeneration::adjust_evacuation_budgets(ShenandoahHeap* heap, Shena byte_size_in_proper_unit(old_evacuated), proper_unit_for_byte_size(old_evacuated), byte_size_in_proper_unit(old_available), proper_unit_for_byte_size(old_available)); - assert(old_available >= old_evacuation_reserve + promotion_reserve + old_bytes_loaned_for_young_evac + allocation_supplement, - "old_available must be larger than accumulated reserves"); - size_t regular_promotion = promotion_reserve - consumed_by_advance_promotion; size_t excess = old_available - (old_evacuation_reserve + promotion_reserve + old_bytes_loaned_for_young_evac + allocation_supplement); @@ -672,7 +796,7 @@ void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) { } { - size_t old_regions_loaned_for_young_evac, regions_available_to_loan, minimum_evacuation_reserve, consumed_by_advance_promotion; + size_t consumed_by_advance_promotion; bool* preselected_regions = nullptr; if (heap->mode()->is_generational()) { preselected_regions = (bool*) alloca(heap->num_regions() * sizeof(bool)); @@ -695,12 +819,10 @@ void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) { // GC is evacuating and updating references. // Budgeting parameters to compute_evacuation_budgets are passed by reference. - compute_evacuation_budgets(heap, preselected_regions, collection_set, old_regions_loaned_for_young_evac, - regions_available_to_loan, minimum_evacuation_reserve, consumed_by_advance_promotion); + compute_evacuation_budgets(heap, preselected_regions, collection_set, consumed_by_advance_promotion); _heuristics->choose_collection_set(collection_set, heap->old_heuristics()); if (!collection_set->is_empty()) { - adjust_evacuation_budgets(heap, collection_set, old_regions_loaned_for_young_evac, regions_available_to_loan, - minimum_evacuation_reserve, consumed_by_advance_promotion); + adjust_evacuation_budgets(heap, collection_set, consumed_by_advance_promotion); } // otherwise, this is an abbreviated cycle and we make no use of evacuation budgets. } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index 75b1d595f5a1c..e59ac8dee09f0 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -61,13 +61,11 @@ class ShenandoahGeneration : public CHeapObj { private: // Compute evacuation budgets prior to choosing collection set. void compute_evacuation_budgets(ShenandoahHeap* heap, bool* preselected_regions, ShenandoahCollectionSet* collection_set, - size_t &old_regions_loaned_for_young_evac, size_t ®ions_available_to_loan, - size_t &minimum_evacuation_reserve, size_t &consumed_by_advance_promotion); + size_t &consumed_by_advance_promotion); // Adjust evacuation budgets after choosing collection set. void adjust_evacuation_budgets(ShenandoahHeap* heap, ShenandoahCollectionSet* collection_set, - size_t old_regions_loaned_for_young_evac, size_t regions_available_to_loan, - size_t minimum_evacuation_reserve, size_t consumed_by_advance_promotion); + size_t consumed_by_advance_promotion); public: ShenandoahGeneration(GenerationMode generation_mode, uint max_workers, size_t max_capacity, size_t soft_max_capacity); From 50c54581270be169eb5c931dc16dfb4f4c8552bc Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Wed, 2 Nov 2022 23:30:07 +0000 Subject: [PATCH 151/254] Fix preemption of coalesce and fill Reviewed-by: wkemper --- .../share/gc/shenandoah/shenandoahOldGeneration.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp index 951c66d043989..d3a378bcd337d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp @@ -219,10 +219,8 @@ void ShenandoahOldGeneration::prepare_gc() { // Make the old generation regions parseable, so they can be safely // scanned when looking for objects in memory indicated by dirty cards. - entry_coalesce_and_fill(); - - // Now that we have made the old generation parseable, it is safe to reset the mark bitmap. - { + if (entry_coalesce_and_fill()) { + // Now that we have made the old generation parseable, it is safe to reset the mark bitmap. static const char* msg = "Concurrent reset (OLD)"; ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_reset_old); ShenandoahWorkerScope scope(ShenandoahHeap::heap()->workers(), @@ -230,6 +228,9 @@ void ShenandoahOldGeneration::prepare_gc() { msg); ShenandoahGeneration::prepare_gc(); } + // Else, coalesce-and-fill has been preempted and we'll finish that effort in the future. Do not invoke + // ShenandoahGeneration::prepare_gc() until coalesce-and-fill is done because it resets the mark bitmap + // and invokes set_mark_incomplete(). Coalesce-and-fill depends on the mark bitmap. } bool ShenandoahOldGeneration::entry_coalesce_and_fill() { From 998f68b26b8d2a5178a30a6c5b596194961e3821 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Mon, 7 Nov 2022 17:37:46 +0000 Subject: [PATCH 152/254] Improve evacuation instrumentation Reviewed-by: kdnilsen --- src/hotspot/share/gc/shared/ageTable.cpp | 50 ++++--- src/hotspot/share/gc/shared/ageTable.hpp | 5 +- .../shenandoahAdaptiveHeuristics.cpp | 22 ++- .../heuristics/shenandoahHeuristics.cpp | 17 ++- .../gc/shenandoah/shenandoahCollectionSet.hpp | 6 +- .../shenandoahCollectionSet.inline.hpp | 2 +- .../gc/shenandoah/shenandoahControlThread.cpp | 6 + .../gc/shenandoah/shenandoahEvacTracker.cpp | 135 ++++++++++++++++++ .../gc/shenandoah/shenandoahEvacTracker.hpp | 71 +++++++++ .../gc/shenandoah/shenandoahGeneration.cpp | 2 +- .../share/gc/shenandoah/shenandoahHeap.cpp | 42 ++++++ .../share/gc/shenandoah/shenandoahHeap.hpp | 10 +- .../gc/shenandoah/shenandoahHeap.inline.hpp | 57 ++------ .../shenandoah/shenandoahThreadLocalData.hpp | 27 +++- 14 files changed, 352 insertions(+), 100 deletions(-) create mode 100644 src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.cpp create mode 100644 src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.hpp diff --git a/src/hotspot/share/gc/shared/ageTable.cpp b/src/hotspot/share/gc/shared/ageTable.cpp index 47b14727bda53..94852e8064d89 100644 --- a/src/hotspot/share/gc/shared/ageTable.cpp +++ b/src/hotspot/share/gc/shared/ageTable.cpp @@ -33,15 +33,16 @@ #include "logging/log.hpp" #include "oops/oop.inline.hpp" #include "utilities/copy.hpp" +#include "logging/logStream.hpp" /* Copyright (c) 1992, 2021, Oracle and/or its affiliates, and Stanford University. See the LICENSE file for license information. */ -AgeTable::AgeTable(bool global) { +AgeTable::AgeTable(bool global) : _use_perf_data(UsePerfData && global) { clear(); - if (UsePerfData && global) { + if (_use_perf_data) { ResourceMark rm; EXCEPTION_MARK; @@ -70,7 +71,7 @@ void AgeTable::clear() { } } -void AgeTable::merge(AgeTable* subTable) { +void AgeTable::merge(const AgeTable* subTable) { for (int i = 0; i < table_size; i++) { sizes[i]+= subTable->sizes[i]; } @@ -105,25 +106,30 @@ uint AgeTable::compute_tenuring_threshold(size_t desired_survivor_size) { } void AgeTable::print_age_table(uint tenuring_threshold) { - if (log_is_enabled(Trace, gc, age) || UsePerfData || AgeTableTracer::is_tenuring_distribution_event_enabled()) { - log_trace(gc, age)("Age table with threshold %u (max threshold " UINTX_FORMAT ")", - tenuring_threshold, MaxTenuringThreshold); - - size_t total = 0; - uint age = 1; - while (age < table_size) { - size_t wordSize = sizes[age]; - total += wordSize; - if (wordSize > 0) { - log_trace(gc, age)("- age %3u: " SIZE_FORMAT_W(10) " bytes, " SIZE_FORMAT_W(10) " total", - age, wordSize * oopSize, total * oopSize); - } - AgeTableTracer::send_tenuring_distribution_event(age, wordSize * oopSize); - if (UsePerfData) { - _perf_sizes[age]->set_value(wordSize * oopSize); - } - age++; - } + LogTarget(Trace, gc, age) lt; + if (lt.is_enabled() || _use_perf_data || AgeTableTracer::is_tenuring_distribution_event_enabled()) { + LogStream st(lt); + print_on(&st, tenuring_threshold); } } +void AgeTable::print_on(outputStream* st, uint tenuring_threshold) { + st->print_cr("Age table with threshold %u (max threshold " UINTX_FORMAT ")", + tenuring_threshold, MaxTenuringThreshold); + + size_t total = 0; + uint age = 1; + while (age < table_size) { + size_t word_size = sizes[age]; + total += word_size; + if (word_size > 0) { + st->print_cr("- age %3u: " SIZE_FORMAT_W(10) " bytes, " SIZE_FORMAT_W(10) " total", + age, word_size * oopSize, total * oopSize); + } + AgeTableTracer::send_tenuring_distribution_event(age, word_size * oopSize); + if (_use_perf_data) { + _perf_sizes[age]->set_value(word_size * oopSize); + } + age++; + } +} \ No newline at end of file diff --git a/src/hotspot/share/gc/shared/ageTable.hpp b/src/hotspot/share/gc/shared/ageTable.hpp index b05e1a161f264..9f0c10ec31203 100644 --- a/src/hotspot/share/gc/shared/ageTable.hpp +++ b/src/hotspot/share/gc/shared/ageTable.hpp @@ -63,14 +63,15 @@ class AgeTable { // Merge another age table with the current one. Used // for parallel young generation gc. - void merge(AgeTable* subTable); + void merge(const AgeTable* subTable); // Calculate new tenuring threshold based on age information. uint compute_tenuring_threshold(size_t desired_survivor_size); void print_age_table(uint tenuring_threshold); + void print_on(outputStream* st, uint tenuring_threshold); private: - + bool _use_perf_data; PerfVariable* _perf_sizes[table_size]; }; diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp index e07315e1555d9..9055d5166155c 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp @@ -112,8 +112,8 @@ void ShenandoahAdaptiveHeuristics::choose_collection_set_from_regiondata(Shenand size_t free_target = (capacity * ShenandoahMinFreeThreshold) / 100 + max_young_cset; size_t min_garbage = (free_target > actual_free) ? (free_target - actual_free) : 0; - log_info(gc, ergo)("Adaptive CSet Selection for GLOBAL. Max Young Cset: " SIZE_FORMAT - "%s, Max Old CSet: " SIZE_FORMAT "%s, Actual Free: " SIZE_FORMAT "%s.", + log_info(gc, ergo)("Adaptive CSet Selection for GLOBAL. Max Young Evacuation: " SIZE_FORMAT + "%s, Max Old Evacuation: " SIZE_FORMAT "%s, Actual Free: " SIZE_FORMAT "%s.", byte_size_in_proper_unit(max_young_cset), proper_unit_for_byte_size(max_young_cset), byte_size_in_proper_unit(max_old_cset), proper_unit_for_byte_size(max_old_cset), byte_size_in_proper_unit(actual_free), proper_unit_for_byte_size(actual_free)); @@ -164,15 +164,12 @@ void ShenandoahAdaptiveHeuristics::choose_collection_set_from_regiondata(Shenand size_t free_target = (capacity * ShenandoahMinFreeThreshold) / 100 + max_cset; size_t min_garbage = (free_target > actual_free) ? (free_target - actual_free) : 0; - log_info(gc, ergo)("Adaptive CSet Selection for YOUNG. Max CSet: " SIZE_FORMAT "%s, Actual Free: " SIZE_FORMAT "%s.", + log_info(gc, ergo)("Adaptive CSet Selection for YOUNG. Max Evacuation: " SIZE_FORMAT "%s, Actual Free: " SIZE_FORMAT "%s.", byte_size_in_proper_unit(max_cset), proper_unit_for_byte_size(max_cset), byte_size_in_proper_unit(actual_free), proper_unit_for_byte_size(actual_free)); for (size_t idx = 0; idx < size; idx++) { ShenandoahHeapRegion* r = data[idx]._region; - size_t new_cset; - size_t region_garbage = r->garbage(); - size_t new_garbage = cur_young_garbage + region_garbage; bool add_region = false; if (!r->is_old()) { @@ -181,14 +178,13 @@ void ShenandoahAdaptiveHeuristics::choose_collection_set_from_regiondata(Shenand // Entire region will be promoted, This region does not impact young-gen evacuation reserve. Memory has already // been set aside to hold evacuation results as advance_promotion_reserve. add_region = true; - new_cset = cur_cset; // Since all live data in this region is being evacuated from young-gen, it is as if this memory // is garbage insofar as young-gen is concerned. Counting this as garbage reduces the need to // reclaim highly utilized young-gen regions just for the sake of finding min_garbage to reclaim // within youn-gen memory cur_young_garbage += r->get_live_data_bytes(); } else if (r->age() < InitialTenuringThreshold) { - new_cset = cur_cset + r->get_live_data_bytes(); + size_t new_cset = cur_cset + r->get_live_data_bytes(); size_t region_garbage = r->garbage(); size_t new_garbage = cur_young_garbage + region_garbage; bool add_regardless = (region_garbage > ignore_threshold) && (new_garbage < min_garbage); @@ -214,7 +210,7 @@ void ShenandoahAdaptiveHeuristics::choose_collection_set_from_regiondata(Shenand size_t free_target = (capacity * ShenandoahMinFreeThreshold) / 100 + max_cset; size_t min_garbage = (free_target > actual_free) ? (free_target - actual_free) : 0; - log_info(gc, ergo)("Adaptive CSet Selection. Max CSet: " SIZE_FORMAT "%s, Actual Free: " SIZE_FORMAT "%s.", + log_info(gc, ergo)("Adaptive CSet Selection. Max Evacuation: " SIZE_FORMAT "%s, Actual Free: " SIZE_FORMAT "%s.", byte_size_in_proper_unit(max_cset), proper_unit_for_byte_size(max_cset), byte_size_in_proper_unit(actual_free), proper_unit_for_byte_size(actual_free)); @@ -363,8 +359,8 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { if (allocation_headroom < min_threshold) { log_info(gc)("Trigger (%s): Free (" SIZE_FORMAT "%s) is below minimum threshold (" SIZE_FORMAT "%s)", _generation->name(), - byte_size_in_proper_unit(available), proper_unit_for_byte_size(available), - byte_size_in_proper_unit(min_threshold), proper_unit_for_byte_size(min_threshold)); + byte_size_in_proper_unit(allocation_headroom), proper_unit_for_byte_size(allocation_headroom), + byte_size_in_proper_unit(min_threshold), proper_unit_for_byte_size(min_threshold)); return true; } @@ -375,8 +371,8 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { if (allocation_headroom < init_threshold) { log_info(gc)("Trigger (%s): Learning " SIZE_FORMAT " of " SIZE_FORMAT ". Free (" SIZE_FORMAT "%s) is below initial threshold (" SIZE_FORMAT "%s)", _generation->name(), _gc_times_learned + 1, max_learn, - byte_size_in_proper_unit(available), proper_unit_for_byte_size(available), - byte_size_in_proper_unit(init_threshold), proper_unit_for_byte_size(init_threshold)); + byte_size_in_proper_unit(allocation_headroom), proper_unit_for_byte_size(allocation_headroom), + byte_size_in_proper_unit(init_threshold), proper_unit_for_byte_size(init_threshold)); return true; } } diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp index 8422256a994b3..5044f0617f55c 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp @@ -232,9 +232,20 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec proper_unit_for_byte_size(collection_set->garbage()), cset_percent); - size_t bytes_evacuated = collection_set->get_bytes_reserved_for_evacuation(); - log_info(gc, ergo)("Total Evacuation: " SIZE_FORMAT "%s", - byte_size_in_proper_unit(bytes_evacuated), proper_unit_for_byte_size(bytes_evacuated)); + if (collection_set->garbage() > 0) { + size_t young_evac_bytes = collection_set->get_young_bytes_reserved_for_evacuation(); + size_t promote_evac_bytes = collection_set->get_young_bytes_to_be_promoted(); + size_t old_evac_bytes = collection_set->get_old_bytes_reserved_for_evacuation(); + size_t total_evac_bytes = young_evac_bytes + promote_evac_bytes + old_evac_bytes; + log_info(gc, ergo)("Evacuation Targets: YOUNG: " SIZE_FORMAT "%s, " + "PROMOTE: " SIZE_FORMAT "%s, " + "OLD: " SIZE_FORMAT "%s, " + "TOTAL: " SIZE_FORMAT "%s", + byte_size_in_proper_unit(young_evac_bytes), proper_unit_for_byte_size(young_evac_bytes), + byte_size_in_proper_unit(promote_evac_bytes), proper_unit_for_byte_size(promote_evac_bytes), + byte_size_in_proper_unit(old_evac_bytes), proper_unit_for_byte_size(old_evac_bytes), + byte_size_in_proper_unit(total_evac_bytes), proper_unit_for_byte_size(total_evac_bytes)); + } } void ShenandoahHeuristics::record_cycle_start() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp index f42a74024238c..6dd5a29dfe3c0 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp @@ -48,9 +48,7 @@ class ShenandoahCollectionSet : public CHeapObj { size_t _used; size_t _region_count; size_t _immediate_trash; - size_t _evacuation_reserve; // How many bytes reserved in generation for evacuation replicas. This does - // not include bytes reserved for old-generation replicas. The value is - // conservative in that memory may be reserved for objects that will be promoted. + size_t _young_bytes_to_evacuate; size_t _young_bytes_to_promote; size_t _old_bytes_to_evacuate; @@ -104,10 +102,8 @@ class ShenandoahCollectionSet : public CHeapObj { // It is not known how many of these bytes will be promoted. inline size_t get_young_bytes_reserved_for_evacuation(); - inline void reserve_young_bytes_for_evacuation(size_t byte_count); inline size_t get_old_bytes_reserved_for_evacuation(); - inline void reserve_old_bytes_for_evacuation(size_t byte_count); inline size_t get_young_bytes_to_be_promoted(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.inline.hpp index 5f877af615cad..0a9c27c45280c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.inline.hpp @@ -66,7 +66,7 @@ size_t ShenandoahCollectionSet::get_old_bytes_reserved_for_evacuation() { } size_t ShenandoahCollectionSet::get_young_bytes_reserved_for_evacuation() { - return _young_bytes_to_evacuate; + return _young_bytes_to_evacuate - _young_bytes_to_promote; } size_t ShenandoahCollectionSet::get_young_bytes_to_be_promoted() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index 8cc9182afe49b..498909bab8459 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -28,6 +28,7 @@ #include "gc/shenandoah/shenandoahConcurrentGC.hpp" #include "gc/shenandoah/shenandoahControlThread.hpp" #include "gc/shenandoah/shenandoahDegeneratedGC.hpp" +#include "gc/shenandoah/shenandoahEvacTracker.hpp" #include "gc/shenandoah/shenandoahFreeSet.hpp" #include "gc/shenandoah/shenandoahFullGC.hpp" #include "gc/shenandoah/shenandoahGeneration.hpp" @@ -398,6 +399,8 @@ void ShenandoahControlThread::process_phase_timings(const ShenandoahHeap* heap) heap->pacer()->flush_stats_to_cycle(); } + ShenandoahCycleStats evac_stats = heap->evac_tracker()->flush_cycle_to_global(); + // Print GC stats for current cycle { LogTarget(Info, gc, stats) lt; @@ -405,6 +408,8 @@ void ShenandoahControlThread::process_phase_timings(const ShenandoahHeap* heap) ResourceMark rm; LogStream ls(lt); heap->phase_timings()->print_cycle_on(&ls); + ShenandoahEvacuationTracker::print_evacuations_on(&ls, &evac_stats.workers, + &evac_stats.mutators); if (ShenandoahPacing) { heap->pacer()->print_cycle_on(&ls); } @@ -413,6 +418,7 @@ void ShenandoahControlThread::process_phase_timings(const ShenandoahHeap* heap) // Commit statistics to globals heap->phase_timings()->flush_cycle_to_global(); + } // Young and old concurrent cycles are initiated by the regulator. Implicit diff --git a/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.cpp b/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.cpp new file mode 100644 index 0000000000000..bd563e4d49861 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.cpp @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2022, Amazon, Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#include "gc/shenandoah/shenandoahEvacTracker.hpp" +#include "gc/shenandoah/shenandoahThreadLocalData.hpp" +#include "gc/shenandoah/shenandoahRootProcessor.hpp" +#include "runtime/threadSMR.inline.hpp" +#include "runtime/thread.hpp" + +ShenandoahEvacuationStats::ShenandoahEvacuationStats() + : _evacuations_completed(0), _bytes_completed(0), + _evacuations_attempted(0), _bytes_attempted(0), + _age_table(false) {} + +void ShenandoahEvacuationStats::begin_evacuation(size_t bytes) { + ++_evacuations_attempted; + _bytes_attempted += bytes; +} + +void ShenandoahEvacuationStats::end_evacuation(size_t bytes, uint age) { + ++_evacuations_completed; + _bytes_completed += bytes; + if (age > 0) { + _age_table.add(age, bytes / oopSize); + } +} + +void ShenandoahEvacuationStats::accumulate(const ShenandoahEvacuationStats* other) { + _evacuations_completed += other->_evacuations_completed; + _bytes_completed += other->_bytes_completed; + _evacuations_attempted += other->_evacuations_attempted; + _bytes_attempted += other->_bytes_attempted; + + _age_table.merge(&other->_age_table); +} + +void ShenandoahEvacuationStats::reset() { + _evacuations_completed = _evacuations_attempted = 0; + _bytes_completed = _bytes_attempted = 0; + _age_table.clear(); +} + +void ShenandoahEvacuationStats::print_on(outputStream* st) { + size_t abandoned_size = _bytes_attempted - _bytes_completed; + size_t abandoned_count = _evacuations_attempted - _evacuations_completed; + st->print_cr("Evacuated " SIZE_FORMAT "%s across " SIZE_FORMAT " objects, " + "abandoned " SIZE_FORMAT "%s across " SIZE_FORMAT " objects.", + byte_size_in_proper_unit(_bytes_completed), + proper_unit_for_byte_size(_bytes_completed), _evacuations_completed, + byte_size_in_proper_unit(abandoned_size), + proper_unit_for_byte_size(abandoned_size), abandoned_count); + _age_table.print_on(st, InitialTenuringThreshold); +} + +void ShenandoahEvacuationTracker::print_global_on(outputStream* st) { + print_evacuations_on(st, &_workers_global, &_mutators_global); +} + +void ShenandoahEvacuationTracker::print_evacuations_on(outputStream* st, + ShenandoahEvacuationStats* workers, + ShenandoahEvacuationStats* mutators) { + st->print("Workers: "); + workers->print_on(st); + st->cr(); + st->print("Mutators: "); + mutators->print_on(st); + st->cr(); + + AgeTable region_ages(false); + ShenandoahHeap* heap = ShenandoahHeap::heap(); + for (uint i = 0; i < heap->num_regions(); ++i) { + ShenandoahHeapRegion* r = heap->get_region(i); + if (r->age() > 0 && r->age() < AgeTable::table_size) { + region_ages.add(r->age(), r->get_live_data_words()); + } + } + st->print("Regions: "); + region_ages.print_on(st, InitialTenuringThreshold); +} + +class ShenandoahStatAggregator : public ThreadClosure { + public: + ShenandoahEvacuationStats* _target; + explicit ShenandoahStatAggregator(ShenandoahEvacuationStats* target) : _target(target) {} + virtual void do_thread(Thread* thread) override { + ShenandoahEvacuationStats* local = ShenandoahThreadLocalData::evacuation_stats(thread); + _target->accumulate(local); + local->reset(); + } +}; + +ShenandoahCycleStats ShenandoahEvacuationTracker::flush_cycle_to_global() { + ShenandoahEvacuationStats mutators, workers; + + ThreadsListHandle java_threads_iterator; + ShenandoahStatAggregator aggregate_mutators(&mutators); + java_threads_iterator.threads_do(&aggregate_mutators); + + ShenandoahStatAggregator aggregate_workers(&workers); + ShenandoahHeap::heap()->gc_threads_do(&aggregate_workers); + + _mutators_global.accumulate(&mutators); + _workers_global.accumulate(&workers); + + return {workers, mutators}; +} + +void ShenandoahEvacuationTracker::begin_evacuation(Thread* thread, size_t bytes) { + ShenandoahThreadLocalData::begin_evacuation(thread, bytes); +} + +void ShenandoahEvacuationTracker::end_evacuation(Thread* thread, size_t bytes, uint age) { + ShenandoahThreadLocalData::end_evacuation(thread, bytes, age); +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.hpp b/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.hpp new file mode 100644 index 0000000000000..18087d762ca98 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.hpp @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2022, Amazon, Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHEVACTRACKER_HPP +#define SHARE_GC_SHENANDOAH_SHENANDOAHEVACTRACKER_HPP + +#include "gc/shared/ageTable.hpp" +#include "utilities/ostream.hpp" + +class ShenandoahEvacuationStats : public CHeapObj { + private: + size_t _evacuations_completed; + size_t _bytes_completed; + size_t _evacuations_attempted; + size_t _bytes_attempted; + + AgeTable _age_table; + + public: + ShenandoahEvacuationStats(); + void begin_evacuation(size_t bytes); + void end_evacuation(size_t bytes, uint age); + + void print_on(outputStream* st); + void accumulate(const ShenandoahEvacuationStats* other); + void reset(); +}; + +struct ShenandoahCycleStats { + ShenandoahEvacuationStats workers; + ShenandoahEvacuationStats mutators; +}; + +class ShenandoahEvacuationTracker : public CHeapObj { + private: + ShenandoahEvacuationStats _workers_global; + ShenandoahEvacuationStats _mutators_global; + + public: + void begin_evacuation(Thread* thread, size_t bytes); + void end_evacuation(Thread* thread, size_t bytes, uint age); + + void print_global_on(outputStream* st); + static void print_evacuations_on(outputStream* st, ShenandoahEvacuationStats* workers, ShenandoahEvacuationStats* mutators); + + ShenandoahCycleStats flush_cycle_to_global(); + private: +}; + +#endif //SHARE_GC_SHENANDOAH_SHENANDOAHEVACTRACKER_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index ac513203999f4..ec5b8657d19c6 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -469,7 +469,7 @@ void ShenandoahGeneration::adjust_evacuation_budgets(ShenandoahHeap* heap, Shena size_t young_promoted = collection_set->get_young_bytes_to_be_promoted(); size_t young_promoted_reserve_used = (size_t) (ShenandoahEvacWaste * young_promoted); - size_t young_evacuated = collection_set->get_young_bytes_reserved_for_evacuation() - young_promoted; + size_t young_evacuated = collection_set->get_young_bytes_reserved_for_evacuation(); size_t young_evacuated_reserve_used = (size_t) (ShenandoahEvacWaste * young_evacuated); // We'll invoke heap->set_young_evac_reserve() further below, after we make additional adjustments to its value diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index d50d620db3fec..9581270245231 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -528,6 +528,7 @@ ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) : _pacer(NULL), _verifier(NULL), _phase_timings(NULL), + _evac_tracker(new ShenandoahEvacuationTracker()), _monitoring_support(NULL), _memory_pool(NULL), _young_gen_memory_pool(NULL), @@ -851,6 +852,43 @@ void ShenandoahHeap::handle_promotion_failure() { old_heuristics()->handle_promotion_failure(); } +void ShenandoahHeap::report_promotion_failure(Thread* thread, size_t size) { + // We squelch excessive reports to reduce noise in logs. Squelch enforcement is not "perfect" because + // this same code can be in-lined in multiple contexts, and each context will have its own copy of the static + // last_report_epoch and this_epoch_report_count variables. + const uint MaxReportsPerEpoch = 4; + static uint last_report_epoch = 0; + static uint epoch_report_count = 0; + + size_t promotion_reserve; + size_t promotion_expended; + + size_t gc_id = control_thread()->get_gc_id(); + + if ((gc_id != last_report_epoch) || (epoch_report_count++ < MaxReportsPerEpoch)) { + { + // Promotion failures should be very rare. Invest in providing useful diagnostic info. + ShenandoahHeapLocker locker(lock()); + promotion_reserve = get_promoted_reserve(); + promotion_expended = get_promoted_expended(); + } + PLAB* plab = ShenandoahThreadLocalData::plab(thread); + size_t words_remaining = (plab == nullptr)? 0: plab->words_remaining(); + const char* promote_enabled = ShenandoahThreadLocalData::allow_plab_promotions(thread)? "enabled": "disabled"; + + log_info(gc, ergo)("Promotion failed, size " SIZE_FORMAT ", has plab? %s, PLAB remaining: " SIZE_FORMAT + ", plab promotions %s, promotion reserve: " SIZE_FORMAT ", promotion expended: " SIZE_FORMAT, + size, plab == nullptr? "no": "yes", + words_remaining, promote_enabled, promotion_reserve, promotion_expended); + if ((gc_id == last_report_epoch) && (epoch_report_count >= MaxReportsPerEpoch)) { + log_info(gc, ergo)("Squelching additional promotion failure reports for epoch %d", last_report_epoch); + } else if (gc_id != last_report_epoch) { + last_report_epoch = gc_id;; + epoch_report_count = 1; + } + } +} + HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) { // New object should fit the GCLAB size size_t min_size = MAX2(size, PLAB::min_size()); @@ -1731,6 +1769,10 @@ void ShenandoahHeap::print_tracing_info() const { shenandoah_policy()->print_gc_stats(&ls); + ls.cr(); + + evac_tracker()->print_global_on(&ls); + ls.cr(); ls.cr(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index ed896b9d6e69f..bddc6634e70e9 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -33,6 +33,7 @@ #include "gc/shenandoah/shenandoahAllocRequest.hpp" #include "gc/shenandoah/shenandoahLock.hpp" #include "gc/shenandoah/shenandoahEvacOOMHandler.hpp" +#include "gc/shenandoah/shenandoahEvacTracker.hpp" #include "gc/shenandoah/shenandoahPadding.hpp" #include "gc/shenandoah/shenandoahSharedVariables.hpp" #include "gc/shenandoah/shenandoahUnload.hpp" @@ -531,7 +532,8 @@ class ShenandoahHeap : public CollectedHeap { ShenandoahPacer* _pacer; ShenandoahVerifier* _verifier; - ShenandoahPhaseTimings* _phase_timings; + ShenandoahPhaseTimings* _phase_timings; + ShenandoahEvacuationTracker* _evac_tracker; ShenandoahControlThread* control_thread() { return _control_thread; } ShenandoahRegulatorThread* regulator_thread() { return _regulator_thread; } @@ -547,7 +549,8 @@ class ShenandoahHeap : public CollectedHeap { ShenandoahFreeSet* free_set() const { return _free_set; } ShenandoahPacer* pacer() const { return _pacer; } - ShenandoahPhaseTimings* phase_timings() const { return _phase_timings; } + ShenandoahPhaseTimings* phase_timings() const { return _phase_timings; } + ShenandoahEvacuationTracker* evac_tracker() const { return _evac_tracker; } ShenandoahVerifier* verifier(); @@ -827,6 +830,7 @@ class ShenandoahHeap : public CollectedHeap { size_t trash_humongous_region_at(ShenandoahHeapRegion *r); static inline void increase_object_age(oop obj, uint additional_age); + static inline uint get_object_age(oop obj); void transfer_old_pointers_from_satb(); private: @@ -839,6 +843,8 @@ class ShenandoahHeap : public CollectedHeap { void try_inject_alloc_failure(); bool should_inject_alloc_failure(); + + void report_promotion_failure(Thread* thread, size_t size); }; #endif // SHARE_GC_SHENANDOAH_SHENANDOAHHEAP_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp index 076dd6c22653b..25176494132d3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp @@ -451,55 +451,7 @@ inline oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, Shenandoah assert(mode()->is_generational(), "Should only be here in generational mode."); if (from_region->is_young()) { // Signal that promotion failed. Will evacuate this old object somewhere in young gen. - - // We squelch excessive reports to reduce noise in logs. Squelch enforcement is not "perfect" because - // this same code can be in-lined in multiple contexts, and each context will have its own copy of the static - // last_report_epoch and this_epoch_report_count variables. - const uint MaxReportsPerEpoch = 4; - static uint last_report_epoch = 0; - static uint epoch_report_count = 0; - PLAB* plab = ShenandoahThreadLocalData::plab(thread); - size_t words_remaining = (plab == nullptr)? 0: plab->words_remaining(); - const char* promote_enabled = ShenandoahThreadLocalData::allow_plab_promotions(thread)? "enabled": "disabled"; - size_t promotion_reserve; - size_t promotion_expended; - // We can only query GCId::current() if current thread is a named thread. If current thread is not a - // named thread, then we don't even try to squelch the promotion failure report, we don't update the - // the last_report_epoch, and we don't increment the epoch_report_count - if (thread->is_Named_thread()) { - uint gc_id = GCId::current(); - if ((gc_id != last_report_epoch) || (epoch_report_count++ < MaxReportsPerEpoch)) { - { - // Promotion failures should be very rare. Invest in providing useful diagnostic info. - ShenandoahHeapLocker locker(lock()); - promotion_reserve = get_promoted_reserve(); - promotion_expended = get_promoted_expended(); - } - log_info(gc, ergo)("Promotion failed, size " SIZE_FORMAT ", has plab? %s, PLAB remaining: " SIZE_FORMAT - ", plab promotions %s, promotion reserve: " SIZE_FORMAT ", promotion expended: " SIZE_FORMAT, - size, plab == nullptr? "no": "yes", - words_remaining, promote_enabled, promotion_reserve, promotion_expended); - if ((gc_id == last_report_epoch) && (epoch_report_count >= MaxReportsPerEpoch)) { - log_info(gc, ergo)("Squelching additional promotion failure reports for epoch %d\n", last_report_epoch); - } else if (gc_id != last_report_epoch) { - last_report_epoch = gc_id;; - epoch_report_count = 1; - } - } - } else if (epoch_report_count < MaxReportsPerEpoch) { - // Unnamed threads are much less common than named threads. In the rare case that an unnamed thread experiences - // a promotion failure before a named thread within a given epoch, the report for the unnamed thread will be squelched. - { - // Promotion failures should be very rare. Invest in providing useful diagnostic info. - ShenandoahHeapLocker locker(lock()); - promotion_reserve = get_promoted_reserve(); - promotion_expended = get_promoted_expended(); - } - log_info(gc, ergo)("Promotion failed (unfiltered), size " SIZE_FORMAT ", has plab? %s, PLAB remaining: " SIZE_FORMAT - ", plab promotions %s, promotion reserve: " SIZE_FORMAT ", promotion expended: " SIZE_FORMAT, - size, plab == nullptr? "no": "yes", - words_remaining, promote_enabled, promotion_reserve, promotion_expended); - } + report_promotion_failure(thread, size); handle_promotion_failure(); return NULL; } else { @@ -517,6 +469,7 @@ inline oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, Shenandoah } // Copy the object: + _evac_tracker->begin_evacuation(thread, size * HeapWordSize); Copy::aligned_disjoint_words(cast_from_oop(p), copy, size); oop copy_val = cast_to_oop(copy); @@ -531,6 +484,7 @@ inline oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, Shenandoah oop result = ShenandoahForwarding::try_update_forwardee(p, copy_val); if (result == copy_val) { // Successfully evacuated. Our copy is now the public one! + _evac_tracker->end_evacuation(thread, size * HeapWordSize, ShenandoahHeap::get_object_age(copy_val)); if (mode()->is_generational() && target_gen == OLD_GENERATION) { handle_old_evacuation(copy, size, from_region->is_young()); } @@ -588,6 +542,11 @@ void ShenandoahHeap::increase_object_age(oop obj, uint additional_age) { } } +uint ShenandoahHeap::get_object_age(oop obj) { + markWord w = obj->has_displaced_mark() ? obj->displaced_mark() : obj->mark(); + return w.age(); +} + inline bool ShenandoahHeap::clear_old_evacuation_failure() { return _old_gen_oom_evac.try_unset(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp b/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp index ff0252e17f755..99664d9e1f2fe 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp @@ -30,6 +30,7 @@ #include "gc/shared/gc_globals.hpp" #include "gc/shenandoah/shenandoahBarrierSet.hpp" #include "gc/shenandoah/shenandoahCodeRoots.hpp" +#include "gc/shenandoah/shenandoahEvacTracker.hpp" #include "gc/shenandoah/shenandoahSATBMarkQueueSet.hpp" #include "runtime/javaThread.hpp" #include "utilities/debug.hpp" @@ -41,13 +42,14 @@ class ShenandoahThreadLocalData { // Evacuation OOM state uint8_t _oom_scope_nesting_level; bool _oom_during_evac; - bool _plab_allows_promotion; // If false, no more promotion by this thread during this evacuation phase. + SATBMarkQueue _satb_mark_queue; // Thread-local allocation buffer for object evacuations. // In generational mode, it is exclusive to the young generation. PLAB* _gclab; size_t _gclab_size; + double _paced_time; // Thread-local allocation buffer only used in generational mode. @@ -60,8 +62,11 @@ class ShenandoahThreadLocalData { size_t _plab_evacuated; size_t _plab_promoted; size_t _plab_preallocated_promoted; + bool _plab_allows_promotion; // If false, no more promotion by this thread during this evacuation phase. bool _plab_retries_enabled; + ShenandoahEvacuationStats* _evacuation_stats; + ShenandoahThreadLocalData() : _gc_state(0), _oom_scope_nesting_level(0), @@ -75,7 +80,9 @@ class ShenandoahThreadLocalData { _plab_evacuated(0), _plab_promoted(0), _plab_preallocated_promoted(0), - _plab_retries_enabled(true) { + _plab_allows_promotion(true), + _plab_retries_enabled(true), + _evacuation_stats(new ShenandoahEvacuationStats()) { } ~ShenandoahThreadLocalData() { @@ -86,6 +93,10 @@ class ShenandoahThreadLocalData { ShenandoahHeap::heap()->retire_plab(_plab); delete _plab; } + + // TODO: Preserve these stats somewhere for mutator threads. + delete _evacuation_stats; + _evacuation_stats = nullptr; } static ShenandoahThreadLocalData* data(Thread* thread) { @@ -139,6 +150,18 @@ class ShenandoahThreadLocalData { data(thread)->_gclab_size = v; } + static void begin_evacuation(Thread* thread, size_t bytes) { + data(thread)->_evacuation_stats->begin_evacuation(bytes); + } + + static void end_evacuation(Thread* thread, size_t bytes, uint age) { + data(thread)->_evacuation_stats->end_evacuation(bytes, age); + } + + static ShenandoahEvacuationStats* evacuation_stats(Thread* thread) { + return data(thread)->_evacuation_stats; + } + static PLAB* plab(Thread* thread) { return data(thread)->_plab; } From 1b110a43ff270d9aa00c4425c049b1f21838d5ee Mon Sep 17 00:00:00 2001 From: William Kemper Date: Thu, 10 Nov 2022 18:44:00 +0000 Subject: [PATCH 153/254] Improve some defaults and remove unused options for generational mode Reviewed-by: rkennke --- .../mode/shenandoahGenerationalMode.cpp | 1 + .../gc/shenandoah/shenandoahHeap.inline.hpp | 49 +++++++++---------- .../gc/shenandoah/shenandoah_globals.hpp | 12 +---- 3 files changed, 27 insertions(+), 35 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp index f33bf97d2fe5b..640eb9b869bd3 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp @@ -36,6 +36,7 @@ void ShenandoahGenerationalMode::initialize_flags() const { FLAG_SET_DEFAULT(VerifyBeforeExit, false); } + SHENANDOAH_ERGO_OVERRIDE_DEFAULT(NewRatio, 1); SHENANDOAH_ERGO_OVERRIDE_DEFAULT(ShenandoahUnloadClassesFrequency, 0); SHENANDOAH_ERGO_ENABLE_FLAG(ExplicitGCInvokesConcurrent); SHENANDOAH_ERGO_ENABLE_FLAG(ShenandoahImplicitGCInvokesConcurrent); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp index 25176494132d3..35f94e8ab187f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp @@ -344,7 +344,7 @@ inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) { ShenandoahRegionAffiliation target_gen = r->affiliation(); if (mode()->is_generational() && ShenandoahHeap::heap()->is_gc_generation_young() && - target_gen == YOUNG_GENERATION && ShenandoahPromoteTenuredObjects) { + target_gen == YOUNG_GENERATION) { markWord mark = p->mark(); if (mark.is_marked()) { // Already forwarded. @@ -394,32 +394,31 @@ inline oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, Shenandoah break; } case OLD_GENERATION: { - if (ShenandoahUsePLAB) { - PLAB* plab = ShenandoahThreadLocalData::plab(thread); - if (plab != nullptr) { - has_plab = true; - } - copy = allocate_from_plab(thread, size, is_promotion); - if ((copy == nullptr) && (size < ShenandoahThreadLocalData::plab_size(thread)) && - ShenandoahThreadLocalData::plab_retries_enabled(thread)) { - // PLAB allocation failed because we are bumping up against the limit on old evacuation reserve or because - // the requested object does not fit within the current plab but the plab still has an "abundance" of memory, - // where abundance is defined as >= PLAB::min_size(). In the former case, we try resetting the desired - // PLAB size and retry PLAB allocation to avoid cascading of shared memory allocations. - - // In this situation, PLAB memory is precious. We'll try to preserve our existing PLAB by forcing - // this particular allocation to be shared. - if (plab->words_remaining() < PLAB::min_size()) { - ShenandoahThreadLocalData::set_plab_size(thread, PLAB::min_size()); - copy = allocate_from_plab(thread, size, is_promotion); - // If we still get nullptr, we'll try a shared allocation below. - if (copy == nullptr) { - // If retry fails, don't continue to retry until we have success (probably in next GC pass) - ShenandoahThreadLocalData::disable_plab_retries(thread); - } + + PLAB* plab = ShenandoahThreadLocalData::plab(thread); + if (plab != nullptr) { + has_plab = true; + } + copy = allocate_from_plab(thread, size, is_promotion); + if ((copy == nullptr) && (size < ShenandoahThreadLocalData::plab_size(thread)) && + ShenandoahThreadLocalData::plab_retries_enabled(thread)) { + // PLAB allocation failed because we are bumping up against the limit on old evacuation reserve or because + // the requested object does not fit within the current plab but the plab still has an "abundance" of memory, + // where abundance is defined as >= PLAB::min_size(). In the former case, we try resetting the desired + // PLAB size and retry PLAB allocation to avoid cascading of shared memory allocations. + + // In this situation, PLAB memory is precious. We'll try to preserve our existing PLAB by forcing + // this particular allocation to be shared. + if (plab->words_remaining() < PLAB::min_size()) { + ShenandoahThreadLocalData::set_plab_size(thread, PLAB::min_size()); + copy = allocate_from_plab(thread, size, is_promotion); + // If we still get nullptr, we'll try a shared allocation below. + if (copy == nullptr) { + // If retry fails, don't continue to retry until we have success (probably in next GC pass) + ShenandoahThreadLocalData::disable_plab_retries(thread); } - // else, copy still equals nullptr. this causes shared allocation below, preserving this plab for future needs. } + // else, copy still equals nullptr. this causes shared allocation below, preserving this plab for future needs. } break; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp index 883cff5dde99e..0befadcc745c0 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp @@ -96,7 +96,7 @@ "collector accepts. In percents of heap region size.") \ range(0,100) \ \ - product(uintx, ShenandoahOldGarbageThreshold, 25, EXPERIMENTAL, \ + product(uintx, ShenandoahOldGarbageThreshold, 10, EXPERIMENTAL, \ "How much garbage an old region has to contain before it would " \ "be taken for collection.") \ range(0,100) \ @@ -152,7 +152,7 @@ "to learn application and GC performance.") \ range(0,100) \ \ - product(uintx, ShenandoahImmediateThreshold, 90, EXPERIMENTAL, \ + product(uintx, ShenandoahImmediateThreshold, 70, EXPERIMENTAL, \ "The cycle may shortcut when enough garbage can be reclaimed " \ "from the immediate garbage (completely garbage regions). " \ "In percents of total garbage found. Setting this threshold " \ @@ -281,10 +281,6 @@ product(bool, ShenandoahElasticTLAB, true, DIAGNOSTIC, \ "Use Elastic TLABs with Shenandoah") \ \ - product(bool, ShenandoahUsePLAB, true, DIAGNOSTIC, \ - "Use PLABs for object promotions with Shenandoah, " \ - "if in generational mode and UseTLAB is also set.") \ - \ product(uintx, ShenandoahEvacReserve, 5, EXPERIMENTAL, \ "How much of (young-generation) heap to reserve for " \ "(young-generation) evacuations. Larger values allow GC to " \ @@ -526,10 +522,6 @@ "memory. Otherwise, old-gen memory cannot be compacted.") \ range(0, 128) \ \ - product(bool, ShenandoahPromoteTenuredObjects, true, DIAGNOSTIC, \ - "Turn on/off evacuating individual tenured young objects " \ - " to the old generation.") \ - \ product(bool, ShenandoahAllowOldMarkingPreemption, true, DIAGNOSTIC, \ "Allow young generation collections to suspend concurrent" \ " marking in the old generation.") \ From 419a3a2ab30f92a15a3c7205dfa0f22502ee69e5 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Wed, 16 Nov 2022 14:45:43 +0000 Subject: [PATCH 154/254] Change affiliation representation Reviewed-by: ysr, wkemper --- .../share/gc/shenandoah/shenandoahAsserts.cpp | 16 ++++ .../share/gc/shenandoah/shenandoahAsserts.hpp | 5 + .../share/gc/shenandoah/shenandoahHeap.cpp | 34 +------ .../share/gc/shenandoah/shenandoahHeap.hpp | 18 +++- .../gc/shenandoah/shenandoahHeap.inline.hpp | 92 +++++++++++++++++++ .../gc/shenandoah/shenandoahHeapRegion.cpp | 22 +++-- .../gc/shenandoah/shenandoahHeapRegion.hpp | 9 +- .../shenandoahHeapRegion.inline.hpp | 12 +++ 8 files changed, 159 insertions(+), 49 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahAsserts.cpp b/src/hotspot/share/gc/shenandoah/shenandoahAsserts.cpp index 62f4de78176a9..a9b276a4a5979 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahAsserts.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahAsserts.cpp @@ -428,3 +428,19 @@ void ShenandoahAsserts::assert_heaplocked_or_safepoint(const char* file, int lin ShenandoahMessageBuffer msg("Heap lock must be owned by current thread, or be at safepoint"); report_vm_error(file, line, msg.buffer()); } + +// unlike assert_heaplocked_or_safepoint(), this does not require current thread in safepoint to be a VM-thread +void ShenandoahAsserts::assert_heaplocked_or_fullgc_safepoint(const char* file, int line) { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + + if (heap->lock()->owned_by_self()) { + return; + } + + if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) { + return; + } + + ShenandoahMessageBuffer msg("Heap lock must be owned by current thread, or be at safepoint"); + report_vm_error(file, line, msg.buffer()); +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahAsserts.hpp b/src/hotspot/share/gc/shenandoah/shenandoahAsserts.hpp index c730eafb89d01..f2b88fea00873 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahAsserts.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahAsserts.hpp @@ -72,6 +72,7 @@ class ShenandoahAsserts { static void assert_heaplocked(const char* file, int line); static void assert_not_heaplocked(const char* file, int line); static void assert_heaplocked_or_safepoint(const char* file, int line); + static void assert_heaplocked_or_fullgc_safepoint(const char* file, int line); #ifdef ASSERT #define shenandoah_assert_in_heap(interior_loc, obj) \ @@ -163,6 +164,9 @@ class ShenandoahAsserts { #define shenandoah_assert_heaplocked_or_safepoint() \ ShenandoahAsserts::assert_heaplocked_or_safepoint(__FILE__, __LINE__) + +#define shenandoah_assert_heaplocked_or_fullgc_safepoint() \ + ShenandoahAsserts::assert_heaplocked_or_fullgc_safepoint(__FILE__, __LINE__) #else #define shenandoah_assert_in_heap(interior_loc, obj) #define shenandoah_assert_in_heap_or_null(interior_loc, obj) @@ -213,6 +217,7 @@ class ShenandoahAsserts { #define shenandoah_assert_heaplocked() #define shenandoah_assert_not_heaplocked() #define shenandoah_assert_heaplocked_or_safepoint() +#define shenandoah_assert_heaplocked_or_fullgc_safepoint() #endif diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 9581270245231..96a49e64420bb 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -353,11 +353,13 @@ jint ShenandoahHeap::initialize() { } _regions = NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, _num_regions, mtGC); + _affiliations = NEW_C_HEAP_ARRAY(uint8_t, _num_regions, mtGC); _free_set = new ShenandoahFreeSet(this, _num_regions); { ShenandoahHeapLocker locker(lock()); + for (size_t i = 0; i < _num_regions; i++) { HeapWord* start = (HeapWord*)sh_rs.base() + ShenandoahHeapRegion::region_size_words() * i; bool is_committed = i < num_committed_regions; @@ -369,6 +371,8 @@ jint ShenandoahHeap::initialize() { _marking_context->initialize_top_at_mark_start(r); _regions[i] = r; assert(!collection_set()->is_in(i), "New region should not be in collection set"); + + _affiliations[i] = ShenandoahRegionAffiliation::FREE; } // Initialize to complete @@ -509,6 +513,7 @@ ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) : _heap_region_special(false), _num_regions(0), _regions(NULL), + _affiliations(NULL), _update_refs_iterator(this), _alloc_supplement_reserve(0), _promoted_reserve(0), @@ -766,35 +771,6 @@ size_t ShenandoahHeap::initial_capacity() const { return _initial_size; } -bool ShenandoahHeap::is_in(const void* p) const { - HeapWord* heap_base = (HeapWord*) base(); - HeapWord* last_region_end = heap_base + ShenandoahHeapRegion::region_size_words() * num_regions(); - return p >= heap_base && p < last_region_end; -} - -bool ShenandoahHeap::is_in_young(const void* p) const { - return is_in(p) && heap_region_containing(p)->affiliation() == ShenandoahRegionAffiliation::YOUNG_GENERATION; -} - -bool ShenandoahHeap::is_in_old(const void* p) const { - return is_in(p) && heap_region_containing(p)->affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION; -} - -bool ShenandoahHeap::is_in_active_generation(oop obj) const { - if (!mode()->is_generational()) { - // everything is the same single generation - return true; - } - - if (active_generation() == NULL) { - // no collection is happening, only expect this to be called - // when concurrent processing is active, but that could change - return false; - } - - return active_generation()->contains(obj); -} - void ShenandoahHeap::op_uncommit(double shrink_before, size_t shrink_until) { assert (ShenandoahUncommit, "should be enabled"); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index bddc6634e70e9..53e8465d89828 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -269,6 +269,7 @@ class ShenandoahHeap : public CollectedHeap { bool _heap_region_special; size_t _num_regions; ShenandoahHeapRegion** _regions; + uint8_t* _affiliations; // Holds array of enum ShenandoahRegionAffiliation, including FREE status in non-generational mode ShenandoahRegionIterator _update_refs_iterator; public: @@ -600,6 +601,9 @@ class ShenandoahHeap : public CollectedHeap { void stw_process_weak_roots(bool full_gc); void stw_weak_refs(bool full_gc); + inline void assert_lock_for_affiliation(ShenandoahRegionAffiliation orig_affiliation, + ShenandoahRegionAffiliation new_affiliation); + // Heap iteration support void scan_roots_for_iteration(ShenandoahScanObjectStack* oop_stack, ObjectIterateScanRootClosure* oops); bool prepare_aux_bitmap_for_iteration(); @@ -614,13 +618,19 @@ class ShenandoahHeap : public CollectedHeap { AdaptiveSizePolicy* size_policy() shenandoah_not_implemented_return(NULL); bool is_maximal_no_gc() const shenandoah_not_implemented_return(false); - bool is_in(const void* p) const; + inline bool is_in(const void* p) const; - bool is_in_active_generation(oop obj) const; - bool is_in_young(const void* p) const; - bool is_in_old(const void* p) const; + inline bool is_in_active_generation(oop obj) const; + inline bool is_in_young(const void* p) const; + inline bool is_in_old(const void* p) const; inline bool is_old(oop pobj) const; + inline ShenandoahRegionAffiliation region_affiliation(const ShenandoahHeapRegion* r); + inline void set_affiliation(ShenandoahHeapRegion* r, ShenandoahRegionAffiliation new_affiliation); + + inline ShenandoahRegionAffiliation region_affiliation(size_t index); + inline void set_affiliation(size_t index, ShenandoahRegionAffiliation new_affiliation); + bool requires_barriers(stackChunkOop obj) const; MemRegion reserved_region() const { return _reserved; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp index 35f94e8ab187f..2e51e93097e03 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp @@ -550,10 +550,102 @@ inline bool ShenandoahHeap::clear_old_evacuation_failure() { return _old_gen_oom_evac.try_unset(); } +bool ShenandoahHeap::is_in(const void* p) const { + HeapWord* heap_base = (HeapWord*) base(); + HeapWord* last_region_end = heap_base + ShenandoahHeapRegion::region_size_words() * num_regions(); + return p >= heap_base && p < last_region_end; +} + +inline bool ShenandoahHeap::is_in_active_generation(oop obj) const { + if (!mode()->is_generational()) { + // everything is the same single generation + return true; + } + + if (active_generation() == NULL) { + // no collection is happening, only expect this to be called + // when concurrent processing is active, but that could change + return false; + } + + assert(is_in(obj), "only check if is in active generation for objects (" PTR_FORMAT ") in heap", p2i(obj)); + assert((active_generation() == (ShenandoahGeneration*) old_generation()) || + (active_generation() == (ShenandoahGeneration*) young_generation()) || + (active_generation() == global_generation()), "Active generation must be old, young, or global"); + + size_t index = heap_region_containing(obj)->index(); + switch (_affiliations[index]) { + case ShenandoahRegionAffiliation::FREE: + // Free regions are in Old, Young, Global + return true; + case ShenandoahRegionAffiliation::YOUNG_GENERATION: + // Young regions are in young_generation and global_generation, not in old_generation + return (active_generation() != (ShenandoahGeneration*) old_generation()); + case ShenandoahRegionAffiliation::OLD_GENERATION: + // Old regions are in old_generation and global_generation, not in young_generation + return (active_generation() != (ShenandoahGeneration*) young_generation()); + default: + assert(false, "Bad affiliation (%d) for region " SIZE_FORMAT, _affiliations[index], index); + return false; + } +} + +inline bool ShenandoahHeap::is_in_young(const void* p) const { + return is_in(p) && (_affiliations[heap_region_index_containing(p)] == ShenandoahRegionAffiliation::YOUNG_GENERATION); +} + +inline bool ShenandoahHeap::is_in_old(const void* p) const { + return is_in(p) && (_affiliations[heap_region_index_containing(p)] == ShenandoahRegionAffiliation::OLD_GENERATION); +} + inline bool ShenandoahHeap::is_old(oop obj) const { return is_gc_generation_young() && is_in_old(obj); } +inline ShenandoahRegionAffiliation ShenandoahHeap::region_affiliation(const ShenandoahHeapRegion *r) { + return (ShenandoahRegionAffiliation) _affiliations[r->index()]; +} + +inline void ShenandoahHeap::assert_lock_for_affiliation(ShenandoahRegionAffiliation orig_affiliation, + ShenandoahRegionAffiliation new_affiliation) { + // A lock is required when changing from FREE to NON-FREE. Though it may be possible to elide the lock when + // transitioning from in-use to FREE, the current implementation uses a lock for this transition. A lock is + // not required to change from YOUNG to OLD (i.e. when promoting humongous region). + // + // new_affiliation is: FREE YOUNG OLD + // orig_affiliation is: FREE X L L + // YOUNG L X + // OLD L X X + // X means state transition won't happen (so don't care) + // L means lock should be held + // Blank means no lock required because affiliation visibility will not be required until subsequent safepoint + // + // Note: during full GC, all transitions between states are possible. During Full GC, we should be in a safepoint. + + if ((orig_affiliation == ShenandoahRegionAffiliation::FREE) || (new_affiliation == ShenandoahRegionAffiliation::FREE)) { + extern bool _is_at_shenandoah_safepoint(); + shenandoah_assert_heaplocked_or_fullgc_safepoint(); + } +} + +inline void ShenandoahHeap::set_affiliation(ShenandoahHeapRegion* r, ShenandoahRegionAffiliation new_affiliation) { +#ifdef ASSERT + assert_lock_for_affiliation(region_affiliation(r), new_affiliation); +#endif + _affiliations[r->index()] = (uint8_t) new_affiliation; +} + +inline ShenandoahRegionAffiliation ShenandoahHeap::region_affiliation(size_t index) { + return (ShenandoahRegionAffiliation) _affiliations[index]; +} + +inline void ShenandoahHeap::set_affiliation(size_t index, ShenandoahRegionAffiliation new_affiliation) { +#ifdef ASSERT + assert_lock_for_affiliation(region_affiliation(index), new_affiliation); +#endif + _affiliations[index] = (uint8_t) new_affiliation; +} + inline bool ShenandoahHeap::requires_marking(const void* entry) const { oop obj = cast_to_oop(entry); return !_marking_context->is_marked_strong(obj); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index 947c565bbb8a7..c8934769d57dc 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -76,7 +76,6 @@ ShenandoahHeapRegion::ShenandoahHeapRegion(HeapWord* start, size_t index, bool c _live_data(0), _critical_pins(0), _update_watermark(start), - _affiliation(FREE), _age(0) { assert(Universe::on_page_boundary(_bottom) && Universe::on_page_boundary(_end), @@ -113,7 +112,8 @@ void ShenandoahHeapRegion::make_regular_allocation(ShenandoahRegionAffiliation a // Change affiliation to YOUNG_GENERATION if _state is not _pinned_cset, _regular, or _pinned. This implements // behavior previously performed as a side effect of make_regular_bypass(). void ShenandoahHeapRegion::make_young_maybe() { - switch (_state) { + shenandoah_assert_heaplocked(); + switch (_state) { case _empty_uncommitted: case _empty_committed: case _cset: @@ -407,7 +407,7 @@ void ShenandoahHeapRegion::print_on(outputStream* st) const { ShouldNotReachHere(); } - switch (_affiliation) { + switch (ShenandoahHeap::heap()->region_affiliation(this)) { case ShenandoahRegionAffiliation::FREE: st->print("|F"); break; @@ -662,6 +662,7 @@ ShenandoahHeapRegion* ShenandoahHeapRegion::humongous_start_region() const { void ShenandoahHeapRegion::recycle() { ShenandoahHeap* heap = ShenandoahHeap::heap(); + shenandoah_assert_heaplocked(); if (affiliation() == YOUNG_GENERATION) { heap->young_generation()->decrease_used(used()); @@ -933,11 +934,12 @@ size_t ShenandoahHeapRegion::pin_count() const { void ShenandoahHeapRegion::set_affiliation(ShenandoahRegionAffiliation new_affiliation) { ShenandoahHeap* heap = ShenandoahHeap::heap(); + ShenandoahRegionAffiliation region_affiliation = heap->region_affiliation(this); { ShenandoahMarkingContext* const ctx = heap->complete_marking_context(); log_debug(gc)("Setting affiliation of Region " SIZE_FORMAT " from %s to %s, top: " PTR_FORMAT ", TAMS: " PTR_FORMAT ", watermark: " PTR_FORMAT ", top_bitmap: " PTR_FORMAT, - index(), affiliation_name(_affiliation), affiliation_name(new_affiliation), + index(), affiliation_name(region_affiliation), affiliation_name(new_affiliation), p2i(top()), p2i(ctx->top_at_mark_start(this)), p2i(_update_watermark), p2i(ctx->top_bitmap(this))); } @@ -954,21 +956,21 @@ void ShenandoahHeapRegion::set_affiliation(ShenandoahRegionAffiliation new_affil } #endif - if (_affiliation == new_affiliation) { + if (region_affiliation == new_affiliation) { return; } if (!heap->mode()->is_generational()) { - _affiliation = new_affiliation; + heap->set_affiliation(this, new_affiliation); return; } log_trace(gc)("Changing affiliation of region %zu from %s to %s", - index(), affiliation_name(_affiliation), affiliation_name(new_affiliation)); + index(), affiliation_name(region_affiliation), affiliation_name(new_affiliation)); - if (_affiliation == ShenandoahRegionAffiliation::YOUNG_GENERATION) { + if (region_affiliation == ShenandoahRegionAffiliation::YOUNG_GENERATION) { heap->young_generation()->decrement_affiliated_region_count(); - } else if (_affiliation == ShenandoahRegionAffiliation::OLD_GENERATION) { + } else if (region_affiliation == ShenandoahRegionAffiliation::OLD_GENERATION) { heap->old_generation()->decrement_affiliated_region_count(); } @@ -987,7 +989,7 @@ void ShenandoahHeapRegion::set_affiliation(ShenandoahRegionAffiliation new_affil ShouldNotReachHere(); return; } - _affiliation = new_affiliation; + heap->set_affiliation(this, new_affiliation); } size_t ShenandoahHeapRegion::promote_humongous() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp index 9c3690d7a10af..101fa2e76e6df 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp @@ -199,8 +199,8 @@ class ShenandoahHeapRegion { bool is_committed() const { return !is_empty_uncommitted(); } bool is_cset() const { return _state == _cset || _state == _pinned_cset; } bool is_pinned() const { return _state == _pinned || _state == _pinned_cset || _state == _pinned_humongous_start; } - bool is_young() const { return _affiliation == YOUNG_GENERATION; } - bool is_old() const { return _affiliation == OLD_GENERATION; } + inline bool is_young() const; + inline bool is_old() const; // Macro-properties: bool is_alloc_allowed() const { return is_empty() || is_regular() || _state == _pinned; } @@ -257,7 +257,6 @@ class ShenandoahHeapRegion { HeapWord* volatile _update_watermark; - ShenandoahRegionAffiliation _affiliation; uint _age; public: @@ -441,9 +440,7 @@ class ShenandoahHeapRegion { inline void set_update_watermark(HeapWord* w); inline void set_update_watermark_at_safepoint(HeapWord* w); - ShenandoahRegionAffiliation affiliation() const { - return _affiliation; - } + inline ShenandoahRegionAffiliation affiliation() const; void set_affiliation(ShenandoahRegionAffiliation new_affiliation); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp index 32738735f33a4..d6c38e6caf60a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp @@ -176,6 +176,10 @@ inline void ShenandoahHeapRegion::set_update_watermark_at_safepoint(HeapWord* w) _update_watermark = w; } +inline ShenandoahRegionAffiliation ShenandoahHeapRegion::affiliation() const { + return ShenandoahHeap::heap()->region_affiliation(this); +} + inline void ShenandoahHeapRegion::clear_young_lab_flags() { _has_young_lab = false; } @@ -188,4 +192,12 @@ inline bool ShenandoahHeapRegion::has_young_lab_flag() { return _has_young_lab; } +inline bool ShenandoahHeapRegion::is_young() const { + return ShenandoahHeap::heap()->region_affiliation(this) == YOUNG_GENERATION; +} + +inline bool ShenandoahHeapRegion::is_old() const { + return ShenandoahHeap::heap()->region_affiliation(this) == OLD_GENERATION; +} + #endif // SHARE_GC_SHENANDOAH_SHENANDOAHHEAPREGION_INLINE_HPP From fe51a9bb089787bc82f96c33b950fe5ac1f4149d Mon Sep 17 00:00:00 2001 From: William Kemper Date: Fri, 18 Nov 2022 19:48:32 +0000 Subject: [PATCH 155/254] Do not apply evacuation budgets in non-generational mode Reviewed-by: kdnilsen --- .../shenandoahAdaptiveHeuristics.cpp | 11 +- .../gc/shenandoah/shenandoahGeneration.cpp | 1002 ++++++++--------- 2 files changed, 502 insertions(+), 511 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp index 9055d5166155c..4225452492adc 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp @@ -205,8 +205,8 @@ void ShenandoahAdaptiveHeuristics::choose_collection_set_from_regiondata(Shenand } } else { // Traditional Shenandoah (non-generational) - size_t max_cset = (size_t) (heap->get_young_evac_reserve() / ShenandoahEvacWaste); - size_t cur_cset = 0; + size_t capacity = ShenandoahHeap::heap()->soft_max_capacity(); + size_t max_cset = (size_t)((1.0 * capacity / 100 * ShenandoahEvacReserve) / ShenandoahEvacWaste); size_t free_target = (capacity * ShenandoahMinFreeThreshold) / 100 + max_cset; size_t min_garbage = (free_target > actual_free) ? (free_target - actual_free) : 0; @@ -214,16 +214,19 @@ void ShenandoahAdaptiveHeuristics::choose_collection_set_from_regiondata(Shenand byte_size_in_proper_unit(max_cset), proper_unit_for_byte_size(max_cset), byte_size_in_proper_unit(actual_free), proper_unit_for_byte_size(actual_free)); + size_t cur_cset = 0; + size_t cur_garbage = 0; + for (size_t idx = 0; idx < size; idx++) { ShenandoahHeapRegion* r = data[idx]._region; size_t new_cset = cur_cset + r->get_live_data_bytes(); size_t region_garbage = r->garbage(); - size_t new_garbage = cur_young_garbage + region_garbage; + size_t new_garbage = cur_garbage + region_garbage; bool add_regardless = (region_garbage > ignore_threshold) && (new_garbage < min_garbage); if ((new_cset <= max_cset) && (add_regardless || (region_garbage > garbage_threshold))) { cset->add_region(r); cur_cset = new_cset; - cur_young_garbage = new_garbage; + cur_garbage = new_garbage; } } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index ec5b8657d19c6..5f906b83cfc2a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -216,193 +216,184 @@ void ShenandoahGeneration::prepare_gc() { void ShenandoahGeneration::compute_evacuation_budgets(ShenandoahHeap* heap, bool* preselected_regions, ShenandoahCollectionSet* collection_set, size_t &consumed_by_advance_promotion) { + assert(heap->mode()->is_generational(), "Only generational mode uses evacuation budgets."); size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); size_t regions_available_to_loan = 0; size_t minimum_evacuation_reserve = ShenandoahOldCompactionReserve * region_size_bytes; size_t old_regions_loaned_for_young_evac = 0; consumed_by_advance_promotion = 0; - if (heap->mode()->is_generational()) { - ShenandoahGeneration* old_generation = heap->old_generation(); - ShenandoahYoungGeneration* young_generation = heap->young_generation(); - size_t avail_evac_reserve_for_loan_to_young_gen = 0; - size_t old_evacuation_reserve = 0; - size_t num_regions = heap->num_regions(); - - // During initialization and phase changes, it is more likely that fewer objects die young and old-gen - // memory is not yet full (or is in the process of being replaced). During these times especially, it - // is beneficial to loan memory from old-gen to young-gen during the evacuation and update-refs phases - // of execution. - - // Calculate EvacuationReserve before PromotionReserve. Evacuation is more critical than promotion. - // If we cannot evacuate old-gen, we will not be able to reclaim old-gen memory. Promotions are less - // critical. If we cannot promote, there may be degradation of young-gen memory because old objects - // accumulate there until they can be promoted. This increases the young-gen marking and evacuation work. - - // Do not fill up old-gen memory with promotions. Reserve some amount of memory for compaction purposes. - ShenandoahOldHeuristics* old_heuristics = heap->old_heuristics(); - size_t young_evac_reserve_max = 0; - if (old_heuristics->unprocessed_old_collection_candidates() > 0) { - // Compute old_evacuation_reserve: how much memory are we reserving to hold the results of - // evacuating old-gen heap regions? In order to sustain a consistent pace of young-gen collections, - // the goal is to maintain a consistent value for this parameter (when the candidate set is not - // empty). This value is the minimum of: - // 1. old_gen->available() - // 2. old-gen->capacity() * ShenandoahOldEvacReserve) / 100 - // (e.g. old evacuation should be no larger than 5% of old_gen capacity) - // 3. ((young_gen->capacity * ShenandoahEvacReserve / 100) * ShenandoahOldEvacRatioPercent) / 100 - // (e.g. old evacuation should be no larger than 12% of young-gen evacuation) - old_evacuation_reserve = old_generation->available(); - assert(old_evacuation_reserve > minimum_evacuation_reserve, "Old-gen available has not been preserved!"); - size_t old_evac_reserve_max = old_generation->soft_max_capacity() * ShenandoahOldEvacReserve / 100; - if (old_evac_reserve_max < old_evacuation_reserve) { - old_evacuation_reserve = old_evac_reserve_max; - } - young_evac_reserve_max = - (((young_generation->soft_max_capacity() * ShenandoahEvacReserve) / 100) * ShenandoahOldEvacRatioPercent) / 100; - if (young_evac_reserve_max < old_evacuation_reserve) { - old_evacuation_reserve = young_evac_reserve_max; - } - } - if (minimum_evacuation_reserve > old_generation->available()) { - // Due to round-off errors during enforcement of minimum_evacuation_reserve during previous GC passes, - // there can be slight discrepancies here. - minimum_evacuation_reserve = old_generation->available(); - } + ShenandoahGeneration* old_generation = heap->old_generation(); + ShenandoahYoungGeneration* young_generation = heap->young_generation(); + size_t old_evacuation_reserve = 0; + size_t num_regions = heap->num_regions(); - heap->set_old_evac_reserve(old_evacuation_reserve); - heap->reset_old_evac_expended(); - - // Compute the young evauation reserve: This is how much memory is available for evacuating young-gen objects. - // We ignore the possible effect of promotions, which reduce demand for young-gen evacuation memory. - // - // TODO: We could give special treatment to the regions that have reached promotion age, because we know their - // live data is entirely eligible for promotion. This knowledge can feed both into calculations of young-gen - // evacuation reserve and promotion reserve. - // - // young_evacuation_reserve for young generation: how much memory are we reserving to hold the results - // of evacuating young collection set regions? This is typically smaller than the total amount - // of available memory, and is also smaller than the total amount of marked live memory within - // young-gen. This value is the smaller of - // - // 1. (young_gen->capacity() * ShenandoahEvacReserve) / 100 - // 2. (young_gen->available() + old_gen_memory_available_to_be_loaned - // - // ShenandoahEvacReserve represents the configured taget size of the evacuation region. We can only honor - // this target if there is memory available to hold the evacuations. Memory is available if it is already - // free within young gen, or if it can be borrowed from old gen. Since we have not yet chosen the collection - // sets, we do not yet know the exact accounting of how many regions will be freed by this collection pass. - // What we do know is that there will be at least one evacuated young-gen region for each old-gen region that - // is loaned to the evacuation effort (because regions to be collected consume more memory than the compacted - // regions that will replace them). In summary, if there are old-gen regions that are available to hold the - // results of young-gen evacuations, it is safe to loan them for this purpose. At this point, we have not yet - // established a promoted_reserve. We'll do that after we choose the collection set and analyze its impact - // on available memory. - // - // We do not know the evacuation_supplement until after we have computed the collection set. It is not always - // the case that young-regions inserted into the collection set will result in net decrease of in-use regions - // because ShenandoahEvacWaste times multiplied by memory within the region may be larger than the region size. - // The problem is especially relevant to regions that have been inserted into the collection set because they have - // reached tenure age. These regions tend to have much higher utilization (e.g. 95%). These regions also offer - // a unique opportunity because we know that every live object contained within the region is elgible to be - // promoted. Thus, the following implementation treats these regions specially: - // - // 1. Before beginning collection set selection, we tally the total amount of live memory held within regions - // that are known to have reached tenure age. If this memory times ShenandoahEvacWaste is available within - // old-gen memory, establish an advance promotion reserve to hold all or some percentage of these objects. - // This advance promotion reserve is excluded from memory available for holding old-gen evacuations and cannot - // be "loaned" to young gen. - // - // 2. Tenure-aged regions are included in the collection set iff their evacuation size * ShenandoahEvacWaste fits - // within the advance promotion reserve. It is counter productive to evacuate these regions if they cannot be - // evacuated directly into old-gen memory. So if there is not sufficient memory to hold copies of their - // live data right now, we'll just let these regions remain in young for now, to be evacuated by a subsequent - // evacuation pass. - // - // 3. Next, we calculate a young-gen evacuation budget, which is the smaller of the two quantities mentioned - // above. old_gen_memory_available_to_be_loaned is calculated as: - // old_gen->available - (advance-promotion-reserve + old-gen_evacuation_reserve) - // - // 4. When choosing the collection set, special care is taken to assure that the amount of loaned memory required to - // hold the results of evacuation is smaller than the total memory occupied by the regions added to the collection - // set. We need to take these precautions because we do not know how much memory will be reclaimed by evacuation - // until after the collection set has been constructed. The algorithm is as follows: - // - // a. We feed into the algorithm (i) young available at the start of evacuation and (ii) the amount of memory - // loaned from old-gen that is available to hold the results of evacuation. - // b. As candidate regions are added into the young-gen collection set, we maintain accumulations of the amount - // of memory spanned by the collection set regions and the amount of memory that must be reserved to hold - // evacuation results (by multiplying live-data size by ShenandoahEvacWaste). We process candidate regions - // in order of decreasing amounts of garbage. We skip over (and do not include into the collection set) any - // regions that do not satisfy all of the following conditions: - // - // i. The amount of live data within the region as scaled by ShenandoahEvacWaste must fit within the - // relevant evacuation reserve (live data of old-gen regions must fit within the old-evac-reserve, live - // data of young-gen tenure-aged regions must fit within the advance promotion reserve, live data within - // other young-gen regions must fit within the youn-gen evacuation reserve). - // ii. The accumulation of memory consumed by evacuation must not exceed the accumulation of memory reclaimed - // through evacuation by more than young-gen available. - // iii. Other conditions may be enforced as appropriate for specific heuristics. - // - // Note that regions are considered for inclusion in the selection set in order of decreasing amounts of garbage. - // It is possible that a region with a larger amount of garbage will be rejected because it also has a larger - // amount of live data and some region that follows this region in candidate order is included in the collection - // set (because it has less live data and thus can fit within the evacuation limits even though it has less - // garbage). - - size_t young_evacuation_reserve = (young_generation->max_capacity() * ShenandoahEvacReserve) / 100; - // old evacuation can pack into existing partially used regions. young evacuation and loans for young allocations - // need to target regions that do not already hold any old-gen objects. Round down. - regions_available_to_loan = old_generation->free_unaffiliated_regions(); - - size_t required_evacuation_reserve; - // Memory evacuated from old-gen on this pass will be available to hold old-gen evacuations in next pass. - if (old_evacuation_reserve > minimum_evacuation_reserve) { - required_evacuation_reserve = 0; - } else { - required_evacuation_reserve = minimum_evacuation_reserve - old_evacuation_reserve; + // During initialization and phase changes, it is more likely that fewer objects die young and old-gen + // memory is not yet full (or is in the process of being replaced). During these times especially, it + // is beneficial to loan memory from old-gen to young-gen during the evacuation and update-refs phases + // of execution. + + // Calculate EvacuationReserve before PromotionReserve. Evacuation is more critical than promotion. + // If we cannot evacuate old-gen, we will not be able to reclaim old-gen memory. Promotions are less + // critical. If we cannot promote, there may be degradation of young-gen memory because old objects + // accumulate there until they can be promoted. This increases the young-gen marking and evacuation work. + + // Do not fill up old-gen memory with promotions. Reserve some amount of memory for compaction purposes. + ShenandoahOldHeuristics* old_heuristics = heap->old_heuristics(); + size_t young_evac_reserve_max = 0; + if (old_heuristics->unprocessed_old_collection_candidates() > 0) { + // Compute old_evacuation_reserve: how much memory are we reserving to hold the results of + // evacuating old-gen heap regions? In order to sustain a consistent pace of young-gen collections, + // the goal is to maintain a consistent value for this parameter (when the candidate set is not + // empty). This value is the minimum of: + // 1. old_gen->available() + // 2. old-gen->capacity() * ShenandoahOldEvacReserve) / 100 + // (e.g. old evacuation should be no larger than 5% of old_gen capacity) + // 3. ((young_gen->capacity * ShenandoahEvacReserve / 100) * ShenandoahOldEvacRatioPercent) / 100 + // (e.g. old evacuation should be no larger than 12% of young-gen evacuation) + old_evacuation_reserve = old_generation->available(); + assert(old_evacuation_reserve > minimum_evacuation_reserve, "Old-gen available has not been preserved!"); + size_t old_evac_reserve_max = old_generation->soft_max_capacity() * ShenandoahOldEvacReserve / 100; + if (old_evac_reserve_max < old_evacuation_reserve) { + old_evacuation_reserve = old_evac_reserve_max; } - - consumed_by_advance_promotion = _heuristics->select_aged_regions( - old_generation->available() - old_evacuation_reserve - required_evacuation_reserve, num_regions, preselected_regions); - size_t net_available_old_regions = - (old_generation->available() - old_evacuation_reserve - consumed_by_advance_promotion) / region_size_bytes; - - if (regions_available_to_loan > net_available_old_regions) { - regions_available_to_loan = net_available_old_regions; + young_evac_reserve_max = + (((young_generation->soft_max_capacity() * ShenandoahEvacReserve) / 100) * ShenandoahOldEvacRatioPercent) / 100; + if (young_evac_reserve_max < old_evacuation_reserve) { + old_evacuation_reserve = young_evac_reserve_max; } + } - // Otherwise, regions_available_to_loan is less than net_available_old_regions because available memory is - // scattered between multiple partially used regions. + if (minimum_evacuation_reserve > old_generation->available()) { + // Due to round-off errors during enforcement of minimum_evacuation_reserve during previous GC passes, + // there can be slight discrepancies here. + minimum_evacuation_reserve = old_generation->available(); + } - if (young_evacuation_reserve > young_generation->available()) { - size_t short_fall = young_evacuation_reserve - young_generation->available(); - if (regions_available_to_loan * region_size_bytes >= short_fall) { - old_regions_loaned_for_young_evac = (short_fall + region_size_bytes - 1) / region_size_bytes; - regions_available_to_loan -= old_regions_loaned_for_young_evac; - } else { - old_regions_loaned_for_young_evac = regions_available_to_loan; - regions_available_to_loan = 0; - young_evacuation_reserve = young_generation->available() + old_regions_loaned_for_young_evac * region_size_bytes; - // In this case, there's no memory available for new allocations while evacuating and updating, unless we - // find more old-gen memory to borrow below. - } - } - // In generational mode, we may end up choosing a young collection set that contains so many promotable objects - // that there is not sufficient space in old generation to hold the promoted objects. That is ok because we have - // assured there is sufficient space in young generation to hold the rejected promotion candidates. These rejected - // promotion candidates will presumably be promoted in a future evacuation cycle. - heap->set_young_evac_reserve(young_evacuation_reserve); - collection_set->establish_preselected(preselected_regions); + heap->set_old_evac_reserve(old_evacuation_reserve); + heap->reset_old_evac_expended(); + + // Compute the young evauation reserve: This is how much memory is available for evacuating young-gen objects. + // We ignore the possible effect of promotions, which reduce demand for young-gen evacuation memory. + // + // TODO: We could give special treatment to the regions that have reached promotion age, because we know their + // live data is entirely eligible for promotion. This knowledge can feed both into calculations of young-gen + // evacuation reserve and promotion reserve. + // + // young_evacuation_reserve for young generation: how much memory are we reserving to hold the results + // of evacuating young collection set regions? This is typically smaller than the total amount + // of available memory, and is also smaller than the total amount of marked live memory within + // young-gen. This value is the smaller of + // + // 1. (young_gen->capacity() * ShenandoahEvacReserve) / 100 + // 2. (young_gen->available() + old_gen_memory_available_to_be_loaned + // + // ShenandoahEvacReserve represents the configured taget size of the evacuation region. We can only honor + // this target if there is memory available to hold the evacuations. Memory is available if it is already + // free within young gen, or if it can be borrowed from old gen. Since we have not yet chosen the collection + // sets, we do not yet know the exact accounting of how many regions will be freed by this collection pass. + // What we do know is that there will be at least one evacuated young-gen region for each old-gen region that + // is loaned to the evacuation effort (because regions to be collected consume more memory than the compacted + // regions that will replace them). In summary, if there are old-gen regions that are available to hold the + // results of young-gen evacuations, it is safe to loan them for this purpose. At this point, we have not yet + // established a promoted_reserve. We'll do that after we choose the collection set and analyze its impact + // on available memory. + // + // We do not know the evacuation_supplement until after we have computed the collection set. It is not always + // the case that young-regions inserted into the collection set will result in net decrease of in-use regions + // because ShenandoahEvacWaste times multiplied by memory within the region may be larger than the region size. + // The problem is especially relevant to regions that have been inserted into the collection set because they have + // reached tenure age. These regions tend to have much higher utilization (e.g. 95%). These regions also offer + // a unique opportunity because we know that every live object contained within the region is elgible to be + // promoted. Thus, the following implementation treats these regions specially: + // + // 1. Before beginning collection set selection, we tally the total amount of live memory held within regions + // that are known to have reached tenure age. If this memory times ShenandoahEvacWaste is available within + // old-gen memory, establish an advance promotion reserve to hold all or some percentage of these objects. + // This advance promotion reserve is excluded from memory available for holding old-gen evacuations and cannot + // be "loaned" to young gen. + // + // 2. Tenure-aged regions are included in the collection set iff their evacuation size * ShenandoahEvacWaste fits + // within the advance promotion reserve. It is counter productive to evacuate these regions if they cannot be + // evacuated directly into old-gen memory. So if there is not sufficient memory to hold copies of their + // live data right now, we'll just let these regions remain in young for now, to be evacuated by a subsequent + // evacuation pass. + // + // 3. Next, we calculate a young-gen evacuation budget, which is the smaller of the two quantities mentioned + // above. old_gen_memory_available_to_be_loaned is calculated as: + // old_gen->available - (advance-promotion-reserve + old-gen_evacuation_reserve) + // + // 4. When choosing the collection set, special care is taken to assure that the amount of loaned memory required to + // hold the results of evacuation is smaller than the total memory occupied by the regions added to the collection + // set. We need to take these precautions because we do not know how much memory will be reclaimed by evacuation + // until after the collection set has been constructed. The algorithm is as follows: + // + // a. We feed into the algorithm (i) young available at the start of evacuation and (ii) the amount of memory + // loaned from old-gen that is available to hold the results of evacuation. + // b. As candidate regions are added into the young-gen collection set, we maintain accumulations of the amount + // of memory spanned by the collection set regions and the amount of memory that must be reserved to hold + // evacuation results (by multiplying live-data size by ShenandoahEvacWaste). We process candidate regions + // in order of decreasing amounts of garbage. We skip over (and do not include into the collection set) any + // regions that do not satisfy all of the following conditions: + // + // i. The amount of live data within the region as scaled by ShenandoahEvacWaste must fit within the + // relevant evacuation reserve (live data of old-gen regions must fit within the old-evac-reserve, live + // data of young-gen tenure-aged regions must fit within the advance promotion reserve, live data within + // other young-gen regions must fit within the youn-gen evacuation reserve). + // ii. The accumulation of memory consumed by evacuation must not exceed the accumulation of memory reclaimed + // through evacuation by more than young-gen available. + // iii. Other conditions may be enforced as appropriate for specific heuristics. + // + // Note that regions are considered for inclusion in the selection set in order of decreasing amounts of garbage. + // It is possible that a region with a larger amount of garbage will be rejected because it also has a larger + // amount of live data and some region that follows this region in candidate order is included in the collection + // set (because it has less live data and thus can fit within the evacuation limits even though it has less + // garbage). + + size_t young_evacuation_reserve = (young_generation->max_capacity() * ShenandoahEvacReserve) / 100; + // old evacuation can pack into existing partially used regions. young evacuation and loans for young allocations + // need to target regions that do not already hold any old-gen objects. Round down. + regions_available_to_loan = old_generation->free_unaffiliated_regions(); + + size_t required_evacuation_reserve; + // Memory evacuated from old-gen on this pass will be available to hold old-gen evacuations in next pass. + if (old_evacuation_reserve > minimum_evacuation_reserve) { + required_evacuation_reserve = 0; } else { - // Not generational mode: limit young evac reserve by young available; no need to establish old_evac_reserve. - ShenandoahYoungGeneration* young_generation = heap->young_generation(); - size_t young_evac_reserve = (young_generation->soft_max_capacity() * ShenandoahEvacReserve) / 100; - if (young_evac_reserve > young_generation->available()) { - young_evac_reserve = young_generation->available(); + required_evacuation_reserve = minimum_evacuation_reserve - old_evacuation_reserve; + } + + consumed_by_advance_promotion = _heuristics->select_aged_regions( + old_generation->available() - old_evacuation_reserve - required_evacuation_reserve, num_regions, preselected_regions); + size_t net_available_old_regions = + (old_generation->available() - old_evacuation_reserve - consumed_by_advance_promotion) / region_size_bytes; + + if (regions_available_to_loan > net_available_old_regions) { + regions_available_to_loan = net_available_old_regions; + } + + // Otherwise, regions_available_to_loan is less than net_available_old_regions because available memory is + // scattered between multiple partially used regions. + + if (young_evacuation_reserve > young_generation->available()) { + size_t short_fall = young_evacuation_reserve - young_generation->available(); + if (regions_available_to_loan * region_size_bytes >= short_fall) { + old_regions_loaned_for_young_evac = (short_fall + region_size_bytes - 1) / region_size_bytes; + regions_available_to_loan -= old_regions_loaned_for_young_evac; + } else { + old_regions_loaned_for_young_evac = regions_available_to_loan; + regions_available_to_loan = 0; + young_evacuation_reserve = young_generation->available() + old_regions_loaned_for_young_evac * region_size_bytes; + // In this case, there's no memory available for new allocations while evacuating and updating, unless we + // find more old-gen memory to borrow below. } - heap->set_young_evac_reserve(young_evac_reserve); } + // In generational mode, we may end up choosing a young collection set that contains so many promotable objects + // that there is not sufficient space in old generation to hold the promoted objects. That is ok because we have + // assured there is sufficient space in young generation to hold the rejected promotion candidates. These rejected + // promotion candidates will presumably be promoted in a future evacuation cycle. + heap->set_young_evac_reserve(young_evacuation_reserve); + collection_set->establish_preselected(preselected_regions); } // Having chosen the collection set, adjust the budgets for generatioal mode based on its composition. Note @@ -425,355 +416,352 @@ void ShenandoahGeneration::adjust_evacuation_budgets(ShenandoahHeap* heap, Shena // available that results from a decrease in memory consumed by old evacuation is not necessarily available to be loaned // to young-gen. - if (heap->mode()->is_generational()) { - size_t old_regions_loaned_for_young_evac, regions_available_to_loan; - size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); - ShenandoahOldGeneration* old_generation = heap->old_generation(); - ShenandoahYoungGeneration* young_generation = heap->young_generation(); - size_t old_evacuated = collection_set->get_old_bytes_reserved_for_evacuation(); - size_t old_evacuated_committed = (size_t) (ShenandoahEvacWaste * old_evacuated); - size_t old_evacuation_reserve = heap->get_old_evac_reserve(); - // Immediate garbage found during choose_collection_set() is all young - size_t immediate_garbage = collection_set->get_immediate_trash(); - size_t old_available = old_generation->available(); - size_t young_available = young_generation->available() + immediate_garbage; - size_t loaned_regions = 0; - size_t available_loan_remnant = 0; // loaned memory that is not yet dedicated to any particular budget - - assert(((consumed_by_advance_promotion * 33) / 32) >= collection_set->get_young_bytes_to_be_promoted() * ShenandoahEvacWaste, - "Advance promotion (" SIZE_FORMAT ") should be at least young_bytes_to_be_promoted (" SIZE_FORMAT - ")* ShenandoahEvacWaste, totalling: " SIZE_FORMAT ", within round-off errors of up to 3.125%%", - consumed_by_advance_promotion, collection_set->get_young_bytes_to_be_promoted(), - (size_t) (collection_set->get_young_bytes_to_be_promoted() * ShenandoahEvacWaste)); - - assert(consumed_by_advance_promotion <= (collection_set->get_young_bytes_to_be_promoted() * ShenandoahEvacWaste * 33) / 32, - "Round-off errors should be less than 3.125%%, consumed by advance: " SIZE_FORMAT ", promoted: " SIZE_FORMAT, - consumed_by_advance_promotion, (size_t) (collection_set->get_young_bytes_to_be_promoted() * ShenandoahEvacWaste)); - - collection_set->abandon_preselected(); - - if (old_evacuated_committed > old_evacuation_reserve) { - // This should only happen due to round-off errors when enforcing ShenandoahEvacWaste - assert(old_evacuated_committed <= (33 * old_evacuation_reserve) / 32, - "Round-off errors should be less than 3.125%%, committed: " SIZE_FORMAT ", reserved: " SIZE_FORMAT, - old_evacuated_committed, old_evacuation_reserve); - old_evacuated_committed = old_evacuation_reserve; - } else if (old_evacuated_committed < old_evacuation_reserve) { - // This may happen if the old-gen collection consumes less than full budget. - old_evacuation_reserve = old_evacuated_committed; - heap->set_old_evac_reserve(old_evacuation_reserve); - } + assert(heap->mode()->is_generational(), "Only generational mode uses evacuation budgets."); + size_t old_regions_loaned_for_young_evac, regions_available_to_loan; + size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); + ShenandoahOldGeneration* old_generation = heap->old_generation(); + ShenandoahYoungGeneration* young_generation = heap->young_generation(); + size_t old_evacuated = collection_set->get_old_bytes_reserved_for_evacuation(); + size_t old_evacuated_committed = (size_t) (ShenandoahEvacWaste * old_evacuated); + size_t old_evacuation_reserve = heap->get_old_evac_reserve(); + // Immediate garbage found during choose_collection_set() is all young + size_t immediate_garbage = collection_set->get_immediate_trash(); + size_t old_available = old_generation->available(); + size_t young_available = young_generation->available() + immediate_garbage; + size_t loaned_regions = 0; + size_t available_loan_remnant = 0; // loaned memory that is not yet dedicated to any particular budget + + assert(((consumed_by_advance_promotion * 33) / 32) >= collection_set->get_young_bytes_to_be_promoted() * ShenandoahEvacWaste, + "Advance promotion (" SIZE_FORMAT ") should be at least young_bytes_to_be_promoted (" SIZE_FORMAT + ")* ShenandoahEvacWaste, totalling: " SIZE_FORMAT ", within round-off errors of up to 3.125%%", + consumed_by_advance_promotion, collection_set->get_young_bytes_to_be_promoted(), + (size_t) (collection_set->get_young_bytes_to_be_promoted() * ShenandoahEvacWaste)); + + assert(consumed_by_advance_promotion <= (collection_set->get_young_bytes_to_be_promoted() * ShenandoahEvacWaste * 33) / 32, + "Round-off errors should be less than 3.125%%, consumed by advance: " SIZE_FORMAT ", promoted: " SIZE_FORMAT, + consumed_by_advance_promotion, (size_t) (collection_set->get_young_bytes_to_be_promoted() * ShenandoahEvacWaste)); + + collection_set->abandon_preselected(); + + if (old_evacuated_committed > old_evacuation_reserve) { + // This should only happen due to round-off errors when enforcing ShenandoahEvacWaste + assert(old_evacuated_committed <= (33 * old_evacuation_reserve) / 32, + "Round-off errors should be less than 3.125%%, committed: " SIZE_FORMAT ", reserved: " SIZE_FORMAT, + old_evacuated_committed, old_evacuation_reserve); + old_evacuated_committed = old_evacuation_reserve; + } else if (old_evacuated_committed < old_evacuation_reserve) { + // This may happen if the old-gen collection consumes less than full budget. + old_evacuation_reserve = old_evacuated_committed; + heap->set_old_evac_reserve(old_evacuation_reserve); + } - // Recompute old_regions_loaned_for_young_evac because young-gen collection set may not need all the memory - // originally reserved. - size_t young_promoted = collection_set->get_young_bytes_to_be_promoted(); - size_t young_promoted_reserve_used = (size_t) (ShenandoahEvacWaste * young_promoted); + // Recompute old_regions_loaned_for_young_evac because young-gen collection set may not need all the memory + // originally reserved. + size_t young_promoted = collection_set->get_young_bytes_to_be_promoted(); + size_t young_promoted_reserve_used = (size_t) (ShenandoahEvacWaste * young_promoted); - size_t young_evacuated = collection_set->get_young_bytes_reserved_for_evacuation(); - size_t young_evacuated_reserve_used = (size_t) (ShenandoahEvacWaste * young_evacuated); + size_t young_evacuated = collection_set->get_young_bytes_reserved_for_evacuation(); + size_t young_evacuated_reserve_used = (size_t) (ShenandoahEvacWaste * young_evacuated); - // We'll invoke heap->set_young_evac_reserve() further below, after we make additional adjustments to its value + // We'll invoke heap->set_young_evac_reserve() further below, after we make additional adjustments to its value - // Adjust old_regions_loaned_for_young_evac to feed into calculations of promoted_reserve - if (young_evacuated_reserve_used > young_available) { - size_t short_fall = young_evacuated_reserve_used - young_available; + // Adjust old_regions_loaned_for_young_evac to feed into calculations of promoted_reserve + if (young_evacuated_reserve_used > young_available) { + size_t short_fall = young_evacuated_reserve_used - young_available; - // region_size_bytes is a power of 2. loan an integral number of regions. - size_t revised_loan_for_young_evacuation = (short_fall + region_size_bytes - 1) / region_size_bytes; + // region_size_bytes is a power of 2. loan an integral number of regions. + size_t revised_loan_for_young_evacuation = (short_fall + region_size_bytes - 1) / region_size_bytes; - // available_loan_remnant represents memory loaned from old-gen but not required for young evacuation. - // This is the excess loaned memory that results from rounding the required loan up to an integral number - // of heap regions. This will be dedicated to alloc_supplement below. - available_loan_remnant = (revised_loan_for_young_evacuation * region_size_bytes) - short_fall; + // available_loan_remnant represents memory loaned from old-gen but not required for young evacuation. + // This is the excess loaned memory that results from rounding the required loan up to an integral number + // of heap regions. This will be dedicated to alloc_supplement below. + available_loan_remnant = (revised_loan_for_young_evacuation * region_size_bytes) - short_fall; - // We previously loaned more than was required by young-gen evacuation. So claw some of this memory back. - old_regions_loaned_for_young_evac = revised_loan_for_young_evacuation; - loaned_regions = old_regions_loaned_for_young_evac; - } else { - // Undo the prevous loan, if any. - old_regions_loaned_for_young_evac = 0; - loaned_regions = 0; - } + // We previously loaned more than was required by young-gen evacuation. So claw some of this memory back. + old_regions_loaned_for_young_evac = revised_loan_for_young_evacuation; + loaned_regions = old_regions_loaned_for_young_evac; + } else { + // Undo the prevous loan, if any. + old_regions_loaned_for_young_evac = 0; + loaned_regions = 0; + } - size_t old_bytes_loaned_for_young_evac = old_regions_loaned_for_young_evac * region_size_bytes - available_loan_remnant; + size_t old_bytes_loaned_for_young_evac = old_regions_loaned_for_young_evac * region_size_bytes - available_loan_remnant; - // Recompute regions_available_to_loan based on possible changes to old_regions_loaned_for_young_evac and - // old_evacuation_reserve. + // Recompute regions_available_to_loan based on possible changes to old_regions_loaned_for_young_evac and + // old_evacuation_reserve. - // Any decrease in old_regions_loaned_for_young_evac are immediately available to be loaned - // However, a change to old_evacuation_reserve() is not necessarily available to loan, because this memory may - // reside within many fragments scattered throughout old-gen. + // Any decrease in old_regions_loaned_for_young_evac are immediately available to be loaned + // However, a change to old_evacuation_reserve() is not necessarily available to loan, because this memory may + // reside within many fragments scattered throughout old-gen. - regions_available_to_loan = old_generation->free_unaffiliated_regions(); - size_t working_old_available = old_generation->available(); + regions_available_to_loan = old_generation->free_unaffiliated_regions(); + size_t working_old_available = old_generation->available(); - assert(regions_available_to_loan * region_size_bytes <= working_old_available, - "Regions available to loan must be less than available memory"); + assert(regions_available_to_loan * region_size_bytes <= working_old_available, + "Regions available to loan must be less than available memory"); - // fragmented_old_total is the amount of memory in old-gen beyond regions_available_to_loan that is otherwise not - // yet dedicated to a particular budget. This memory can be used for promotion_reserve. - size_t fragmented_old_total = working_old_available - regions_available_to_loan * region_size_bytes; + // fragmented_old_total is the amount of memory in old-gen beyond regions_available_to_loan that is otherwise not + // yet dedicated to a particular budget. This memory can be used for promotion_reserve. + size_t fragmented_old_total = working_old_available - regions_available_to_loan * region_size_bytes; - // fragmented_old_usage is the memory that is dedicated to holding evacuated old-gen objects, which does not need - // to be an integral number of regions. - size_t fragmented_old_usage = old_evacuated_committed + consumed_by_advance_promotion; + // fragmented_old_usage is the memory that is dedicated to holding evacuated old-gen objects, which does not need + // to be an integral number of regions. + size_t fragmented_old_usage = old_evacuated_committed + consumed_by_advance_promotion; - if (fragmented_old_total >= fragmented_old_usage) { - // Seems this will be rare. In this case, all of the memory required for old-gen evacuations and promotions can be - // taken from the existing fragments within old-gen. Reduce this fragmented total by this amount. - fragmented_old_total -= fragmented_old_usage; - // And reduce regions_available_to_loan by the regions dedicated to young_evac. - regions_available_to_loan -= old_regions_loaned_for_young_evac; - } else { - // In this case, we need to dedicate some of the regions_available_to_loan to hold the results of old-gen evacuations - // and promotions. + if (fragmented_old_total >= fragmented_old_usage) { + // Seems this will be rare. In this case, all of the memory required for old-gen evacuations and promotions can be + // taken from the existing fragments within old-gen. Reduce this fragmented total by this amount. + fragmented_old_total -= fragmented_old_usage; + // And reduce regions_available_to_loan by the regions dedicated to young_evac. + regions_available_to_loan -= old_regions_loaned_for_young_evac; + } else { + // In this case, we need to dedicate some of the regions_available_to_loan to hold the results of old-gen evacuations + // and promotions. - size_t unaffiliated_memory_required_for_old = fragmented_old_usage - fragmented_old_total; - size_t unaffiliated_regions_used_by_old = (unaffiliated_memory_required_for_old + region_size_bytes - 1) / region_size_bytes; - regions_available_to_loan -= (unaffiliated_regions_used_by_old + old_regions_loaned_for_young_evac); + size_t unaffiliated_memory_required_for_old = fragmented_old_usage - fragmented_old_total; + size_t unaffiliated_regions_used_by_old = (unaffiliated_memory_required_for_old + region_size_bytes - 1) / region_size_bytes; + regions_available_to_loan -= (unaffiliated_regions_used_by_old + old_regions_loaned_for_young_evac); - size_t memory_for_promotions_and_old_evac = fragmented_old_total + unaffiliated_regions_used_by_old; - size_t memory_required_for_promotions_and_old_evac = fragmented_old_usage; - size_t excess_fragmented = memory_for_promotions_and_old_evac - memory_required_for_promotions_and_old_evac; - fragmented_old_total = excess_fragmented; + size_t memory_for_promotions_and_old_evac = fragmented_old_total + unaffiliated_regions_used_by_old; + size_t memory_required_for_promotions_and_old_evac = fragmented_old_usage; + size_t excess_fragmented = memory_for_promotions_and_old_evac - memory_required_for_promotions_and_old_evac; + fragmented_old_total = excess_fragmented; + } + + // Subtract from working_old_available old_evacuated_committed and consumed_by_advance_promotion + working_old_available -= fragmented_old_usage; + // And also subtract out the regions loaned for young evacuation + working_old_available -= old_regions_loaned_for_young_evac * region_size_bytes; + + // Assure that old_evacuated_committed + old_bytes_loaned_for_young_evac >= the minimum evacuation reserve + // in order to prevent promotion reserve from violating minimum evacuation reserve. + size_t old_regions_reserved_for_alloc_supplement = 0; + size_t old_bytes_reserved_for_alloc_supplement = 0; + size_t reserved_bytes_for_future_old_evac = 0; + + old_bytes_reserved_for_alloc_supplement = available_loan_remnant; + available_loan_remnant = 0; + + // Memory that has been loaned for young evacuations and old-gen regions in the current mixed-evacuation collection + // set will be available to hold future old-gen evacuations. If this memory is less than the desired amount of memory + // set aside for old-gen compaction reserve, try to set aside additional memory so that it will be available during + // the next mixed evacuation cycle. Note that memory loaned to young-gen for allocation supplement is excluded from + // the old-gen promotion reserve. + size_t future_evac_reserve_regions = old_regions_loaned_for_young_evac + collection_set->get_old_region_count(); + size_t collected_regions = collection_set->get_young_region_count(); + + if (future_evac_reserve_regions < ShenandoahOldCompactionReserve) { + // Require that we loan more memory for holding young evacuations to assure that we have adequate reserves to receive + // old-gen evacuations during subsequent collections. Loaning this memory for an allocation supplement does not + // satisfy our needs because newly allocated objects are not necessarily counter-balanced by reclaimed collection + // set regions. + + // Put this memory into reserve by identifying it as old_regions_loaned_for_young_evac + size_t additional_regions_to_loan = ShenandoahOldCompactionReserve - future_evac_reserve_regions; + + // We can loan additional regions to be repaid from the anticipated recycling of young collection set regions + // provided that these regions are currently available within old-gen memory. + size_t collected_regions_to_loan; + if (collected_regions >= additional_regions_to_loan) { + collected_regions_to_loan = additional_regions_to_loan; + additional_regions_to_loan = 0; + } else if (collected_regions > 0) { + collected_regions_to_loan = collected_regions; + additional_regions_to_loan -= collected_regions_to_loan; + } else { + collected_regions_to_loan = 0; + } + + if (collected_regions_to_loan > 0) { + // We're evacuating at least this many regions, it's ok to use these regions for allocation supplement since + // we'll be able to repay the loan at end of this GC pass, assuming the regions are available. + if (collected_regions_to_loan > regions_available_to_loan) { + collected_regions_to_loan = regions_available_to_loan; + } + old_bytes_reserved_for_alloc_supplement += collected_regions_to_loan * region_size_bytes; + regions_available_to_loan -= collected_regions_to_loan; + loaned_regions += collected_regions_to_loan; + working_old_available -= collected_regions_to_loan * region_size_bytes; } - // Subtract from working_old_available old_evacuated_committed and consumed_by_advance_promotion - working_old_available -= fragmented_old_usage; - // And also subtract out the regions loaned for young evacuation - working_old_available -= old_regions_loaned_for_young_evac * region_size_bytes; - - // Assure that old_evacuated_committed + old_bytes_loaned_for_young_evac >= the minimum evacuation reserve - // in order to prevent promotion reserve from violating minimum evacuation reserve. - size_t old_regions_reserved_for_alloc_supplement = 0; - size_t old_bytes_reserved_for_alloc_supplement = 0; - size_t reserved_bytes_for_future_old_evac = 0; - - old_bytes_reserved_for_alloc_supplement = available_loan_remnant; - available_loan_remnant = 0; - - // Memory that has been loaned for young evacuations and old-gen regions in the current mixed-evacuation collection - // set will be available to hold future old-gen evacuations. If this memory is less than the desired amount of memory - // set aside for old-gen compaction reserve, try to set aside additional memory so that it will be available during - // the next mixed evacuation cycle. Note that memory loaned to young-gen for allocation supplement is excluded from - // the old-gen promotion reserve. - size_t future_evac_reserve_regions = old_regions_loaned_for_young_evac + collection_set->get_old_region_count(); - size_t collected_regions = collection_set->get_young_region_count(); - - if (future_evac_reserve_regions < ShenandoahOldCompactionReserve) { - // Require that we loan more memory for holding young evacuations to assure that we have adequate reserves to receive - // old-gen evacuations during subsequent collections. Loaning this memory for an allocation supplement does not - // satisfy our needs because newly allocated objects are not necessarily counter-balanced by reclaimed collection - // set regions. - - // Put this memory into reserve by identifying it as old_regions_loaned_for_young_evac - size_t additional_regions_to_loan = ShenandoahOldCompactionReserve - future_evac_reserve_regions; - - // We can loan additional regions to be repaid from the anticipated recycling of young collection set regions - // provided that these regions are currently available within old-gen memory. - size_t collected_regions_to_loan; - if (collected_regions >= additional_regions_to_loan) { - collected_regions_to_loan = additional_regions_to_loan; - additional_regions_to_loan = 0; - } else if (collected_regions > 0) { - collected_regions_to_loan = collected_regions; - additional_regions_to_loan -= collected_regions_to_loan; + // If there's still memory that we want to exclude from the current promotion reserve, but we are unable to loan + // this memory because fully empty old-gen regions are not available, decrement the working_old_available to make + // sure that this memory is not used to hold the results of old-gen evacuation. + if (additional_regions_to_loan > regions_available_to_loan) { + size_t unloaned_regions = additional_regions_to_loan - regions_available_to_loan; + size_t unloaned_bytes = unloaned_regions * region_size_bytes; + + if (working_old_available < unloaned_bytes) { + // We're in dire straits. We won't be able to reserve all the memory that we want to make available for the + // next old-gen evacuation. We'll reserve as much of it as possible. Setting working_old_available to zero + // means there will be no promotion except for the advance promotion. Note that if some advance promotion fails, + // the object will be evacuated to young-gen so we should still end up reclaiming the entire advance promotion + // collection set. + reserved_bytes_for_future_old_evac = working_old_available; + working_old_available = 0; } else { - collected_regions_to_loan = 0; + reserved_bytes_for_future_old_evac = unloaned_bytes; + working_old_available -= unloaned_bytes; } + size_t regions_reserved_for_future_old_evac = + (reserved_bytes_for_future_old_evac + region_size_bytes - 1) / region_size_bytes; - if (collected_regions_to_loan > 0) { - // We're evacuating at least this many regions, it's ok to use these regions for allocation supplement since - // we'll be able to repay the loan at end of this GC pass, assuming the regions are available. - if (collected_regions_to_loan > regions_available_to_loan) { - collected_regions_to_loan = regions_available_to_loan; - } - old_bytes_reserved_for_alloc_supplement += collected_regions_to_loan * region_size_bytes; - regions_available_to_loan -= collected_regions_to_loan; - loaned_regions += collected_regions_to_loan; - working_old_available -= collected_regions_to_loan * region_size_bytes; + if (regions_reserved_for_future_old_evac < regions_available_to_loan) { + regions_available_to_loan -= regions_reserved_for_future_old_evac; + } else { + regions_available_to_loan = 0; } - // If there's still memory that we want to exclude from the current promotion reserve, but we are unable to loan - // this memory because fully empty old-gen regions are not available, decrement the working_old_available to make - // sure that this memory is not used to hold the results of old-gen evacuation. - if (additional_regions_to_loan > regions_available_to_loan) { - size_t unloaned_regions = additional_regions_to_loan - regions_available_to_loan; - size_t unloaned_bytes = unloaned_regions * region_size_bytes; - - if (working_old_available < unloaned_bytes) { - // We're in dire straits. We won't be able to reserve all the memory that we want to make available for the - // next old-gen evacuation. We'll reserve as much of it as possible. Setting working_old_available to zero - // means there will be no promotion except for the advance promotion. Note that if some advance promotion fails, - // the object will be evacuated to young-gen so we should still end up reclaiming the entire advance promotion - // collection set. - reserved_bytes_for_future_old_evac = working_old_available; - working_old_available = 0; - } else { - reserved_bytes_for_future_old_evac = unloaned_bytes; - working_old_available -= unloaned_bytes; - } - size_t regions_reserved_for_future_old_evac = - (reserved_bytes_for_future_old_evac + region_size_bytes - 1) / region_size_bytes; - - if (regions_reserved_for_future_old_evac < regions_available_to_loan) { - regions_available_to_loan -= regions_reserved_for_future_old_evac; - } else { - regions_available_to_loan = 0; - } - - // Since we're in dire straits, zero out fragmented_old_total so this won't be used for promotion; - if (working_old_available > fragmented_old_total) { - working_old_available -= fragmented_old_total; - } else { - working_old_available = 0; - } - fragmented_old_total = 0; + // Since we're in dire straits, zero out fragmented_old_total so this won't be used for promotion; + if (working_old_available > fragmented_old_total) { + working_old_available -= fragmented_old_total; + } else { + working_old_available = 0; } + fragmented_old_total = 0; } + } - // Establish young_evac_reserve so that this young-gen memory is not used for new allocations, allowing the memory - // to be returned to old-gen as soon as the current collection set regions are reclaimed. - heap->set_young_evac_reserve(young_evacuated_reserve_used); - - // Limit promoted_reserve so that we can set aside memory to be loaned from old-gen to young-gen. This - // value is not "critical". If we underestimate, certain promotions will simply be deferred. If we put - // "all the rest" of old-gen memory into the promotion reserve, we'll have nothing left to loan to young-gen - // during the evac and update phases of GC. So we "limit" the sizes of the promotion budget to be the smaller of: - // - // 1. old_available - // (old_available is old_gen->available() - - // (old_evacuated_committed + consumed_by_advance_promotion + loaned_for_young_evac + reserved_for_alloc_supplement)) - // 2. young bytes reserved for evacuation (we can't promote more than young is evacuating) - size_t promotion_reserve = working_old_available; - - // We experimented with constraining promoted_reserve to be no larger than 4 times the size of previously_promoted, - // but this constraint was too limiting, resulting in failure of legitimate promotions. This was tried before we - // had special handling in place for advance promotion. We should retry now that advance promotion is handled - // specially. - - // We had also experimented with constraining promoted_reserve to be no more than young_evacuation_committed - // divided by promotion_divisor, where: - // size_t promotion_divisor = (0x02 << InitialTenuringThreshold) - 1; - // This also was found to be too limiting, resulting in failure of legitimate promotions. - // - // Both experiments were conducted in the presence of other bugs which could have been the root cause for - // the failures identified above as being "too limiting". TODO: conduct new experiments with the more limiting - // values of young_evacuation_reserved_used. - - // young_evacuation_reserve_used already excludes bytes known to be promoted, which equals consumed_by_advance_promotion - if (young_evacuated_reserve_used < promotion_reserve) { - // Shrink promotion_reserve if it is larger than the memory to be consumed by evacuating all young objects in - // collection set, including anticipated waste. There's no benefit in using a larger promotion_reserve. - // young_evacuation_reserve_used does not include live memory within tenure-aged regions. - promotion_reserve = young_evacuated_reserve_used; - } - assert(working_old_available >= promotion_reserve, "Cannot reserve for promotion more than is available"); - working_old_available -= promotion_reserve; - // Having reserved this memory for promotion, the regions are no longer available to be loaned. - size_t regions_consumed_by_promotion_reserve = (promotion_reserve + region_size_bytes - 1) / region_size_bytes; - if (regions_consumed_by_promotion_reserve > regions_available_to_loan) { - // This can happen if the promotion reserve makes use of memory that is fragmented between many partially available - // old-gen regions. - regions_available_to_loan = 0; - } else { - regions_available_to_loan -= regions_consumed_by_promotion_reserve; - } + // Establish young_evac_reserve so that this young-gen memory is not used for new allocations, allowing the memory + // to be returned to old-gen as soon as the current collection set regions are reclaimed. + heap->set_young_evac_reserve(young_evacuated_reserve_used); + + // Limit promoted_reserve so that we can set aside memory to be loaned from old-gen to young-gen. This + // value is not "critical". If we underestimate, certain promotions will simply be deferred. If we put + // "all the rest" of old-gen memory into the promotion reserve, we'll have nothing left to loan to young-gen + // during the evac and update phases of GC. So we "limit" the sizes of the promotion budget to be the smaller of: + // + // 1. old_available + // (old_available is old_gen->available() - + // (old_evacuated_committed + consumed_by_advance_promotion + loaned_for_young_evac + reserved_for_alloc_supplement)) + // 2. young bytes reserved for evacuation (we can't promote more than young is evacuating) + size_t promotion_reserve = working_old_available; + + // We experimented with constraining promoted_reserve to be no larger than 4 times the size of previously_promoted, + // but this constraint was too limiting, resulting in failure of legitimate promotions. This was tried before we + // had special handling in place for advance promotion. We should retry now that advance promotion is handled + // specially. + + // We had also experimented with constraining promoted_reserve to be no more than young_evacuation_committed + // divided by promotion_divisor, where: + // size_t promotion_divisor = (0x02 << InitialTenuringThreshold) - 1; + // This also was found to be too limiting, resulting in failure of legitimate promotions. + // + // Both experiments were conducted in the presence of other bugs which could have been the root cause for + // the failures identified above as being "too limiting". TODO: conduct new experiments with the more limiting + // values of young_evacuation_reserved_used. + + // young_evacuation_reserve_used already excludes bytes known to be promoted, which equals consumed_by_advance_promotion + if (young_evacuated_reserve_used < promotion_reserve) { + // Shrink promotion_reserve if it is larger than the memory to be consumed by evacuating all young objects in + // collection set, including anticipated waste. There's no benefit in using a larger promotion_reserve. + // young_evacuation_reserve_used does not include live memory within tenure-aged regions. + promotion_reserve = young_evacuated_reserve_used; + } + assert(working_old_available >= promotion_reserve, "Cannot reserve for promotion more than is available"); + working_old_available -= promotion_reserve; + // Having reserved this memory for promotion, the regions are no longer available to be loaned. + size_t regions_consumed_by_promotion_reserve = (promotion_reserve + region_size_bytes - 1) / region_size_bytes; + if (regions_consumed_by_promotion_reserve > regions_available_to_loan) { + // This can happen if the promotion reserve makes use of memory that is fragmented between many partially available + // old-gen regions. + regions_available_to_loan = 0; + } else { + regions_available_to_loan -= regions_consumed_by_promotion_reserve; + } - log_debug(gc)("old_gen->available(): " SIZE_FORMAT " divided between promotion reserve: " SIZE_FORMAT - ", old evacuation reserve: " SIZE_FORMAT ", advance promotion reserve supplement: " SIZE_FORMAT - ", old loaned for young evacuation: " SIZE_FORMAT ", old reserved for alloc supplement: " SIZE_FORMAT, - old_generation->available(), promotion_reserve, old_evacuated_committed, consumed_by_advance_promotion, - old_regions_loaned_for_young_evac * region_size_bytes, old_bytes_reserved_for_alloc_supplement); + log_debug(gc)("old_gen->available(): " SIZE_FORMAT " divided between promotion reserve: " SIZE_FORMAT + ", old evacuation reserve: " SIZE_FORMAT ", advance promotion reserve supplement: " SIZE_FORMAT + ", old loaned for young evacuation: " SIZE_FORMAT ", old reserved for alloc supplement: " SIZE_FORMAT, + old_generation->available(), promotion_reserve, old_evacuated_committed, consumed_by_advance_promotion, + old_regions_loaned_for_young_evac * region_size_bytes, old_bytes_reserved_for_alloc_supplement); - promotion_reserve += consumed_by_advance_promotion; - heap->set_promoted_reserve(promotion_reserve); + promotion_reserve += consumed_by_advance_promotion; + heap->set_promoted_reserve(promotion_reserve); - heap->reset_promoted_expended(); - if (collection_set->get_old_bytes_reserved_for_evacuation() == 0) { - // Setting old evacuation reserve to zero denotes that there is no old-gen evacuation in this pass. - heap->set_old_evac_reserve(0); - } + heap->reset_promoted_expended(); + if (collection_set->get_old_bytes_reserved_for_evacuation() == 0) { + // Setting old evacuation reserve to zero denotes that there is no old-gen evacuation in this pass. + heap->set_old_evac_reserve(0); + } - size_t old_gen_usage_base = old_generation->used() - collection_set->get_old_garbage(); - heap->capture_old_usage(old_gen_usage_base); - - // Compute additional evacuation supplement, which is extra memory borrowed from old-gen that can be allocated - // by mutators while GC is working on evacuation and update-refs. This memory can be temporarily borrowed - // from old-gen allotment, then repaid at the end of update-refs from the recycled collection set. After - // we have computed the collection set based on the parameters established above, we can make additional - // loans based on our knowledge of the collection set to determine how much allocation we can allow - // during the evacuation and update-refs phases of execution. The total available supplement is the result - // of adding old_bytes_reserved_for_alloc_supplement to the smaller of: - // - // 1. regions_available_to_loan * region_size_bytes - // 2. The replenishment budget (number of regions in collection set - the number of regions already - // under lien for the young_evacuation_reserve) - // - - // Regardless of how many regions may be available to be loaned, we can loan no more regions than - // the total number of young regions to be evacuated. Call this the regions_for_runway. - - if (regions_available_to_loan > 0 && (collected_regions > loaned_regions)) { - assert(regions_available_to_loan * region_size_bytes <= working_old_available, - "regions_available_to_loan should not exceed working_old_available"); - - size_t additional_regions_to_loan = collected_regions - loaned_regions; - if (additional_regions_to_loan > regions_available_to_loan) { - additional_regions_to_loan = regions_available_to_loan; - } - loaned_regions += additional_regions_to_loan; - old_bytes_reserved_for_alloc_supplement += additional_regions_to_loan * region_size_bytes; - working_old_available -= additional_regions_to_loan * region_size_bytes; + size_t old_gen_usage_base = old_generation->used() - collection_set->get_old_garbage(); + heap->capture_old_usage(old_gen_usage_base); + + // Compute additional evacuation supplement, which is extra memory borrowed from old-gen that can be allocated + // by mutators while GC is working on evacuation and update-refs. This memory can be temporarily borrowed + // from old-gen allotment, then repaid at the end of update-refs from the recycled collection set. After + // we have computed the collection set based on the parameters established above, we can make additional + // loans based on our knowledge of the collection set to determine how much allocation we can allow + // during the evacuation and update-refs phases of execution. The total available supplement is the result + // of adding old_bytes_reserved_for_alloc_supplement to the smaller of: + // + // 1. regions_available_to_loan * region_size_bytes + // 2. The replenishment budget (number of regions in collection set - the number of regions already + // under lien for the young_evacuation_reserve) + // + + // Regardless of how many regions may be available to be loaned, we can loan no more regions than + // the total number of young regions to be evacuated. Call this the regions_for_runway. + + if (regions_available_to_loan > 0 && (collected_regions > loaned_regions)) { + assert(regions_available_to_loan * region_size_bytes <= working_old_available, + "regions_available_to_loan should not exceed working_old_available"); + + size_t additional_regions_to_loan = collected_regions - loaned_regions; + if (additional_regions_to_loan > regions_available_to_loan) { + additional_regions_to_loan = regions_available_to_loan; } - size_t allocation_supplement = old_bytes_reserved_for_alloc_supplement; - heap->set_alloc_supplement_reserve(allocation_supplement); - - // TODO: young_available, which feeds into alloc_budget_evac_and_update is lacking memory available within - // existing young-gen regions that were not selected for the collection set. Add this in and adjust the - // log message (where it says "empty-region allocation budget"). - - log_debug(gc)("Memory reserved for young evacuation: " SIZE_FORMAT "%s for evacuating " SIZE_FORMAT - "%s out of young available: " SIZE_FORMAT "%s", - byte_size_in_proper_unit(young_evacuated_reserve_used), - proper_unit_for_byte_size(young_evacuated_reserve_used), - byte_size_in_proper_unit(young_evacuated), proper_unit_for_byte_size(young_evacuated), - byte_size_in_proper_unit(young_available), proper_unit_for_byte_size(young_available)); - - log_debug(gc)("Memory reserved for old evacuation: " SIZE_FORMAT "%s for evacuating " SIZE_FORMAT - "%s out of old available: " SIZE_FORMAT "%s", - byte_size_in_proper_unit(old_evacuated), proper_unit_for_byte_size(old_evacuated), - byte_size_in_proper_unit(old_evacuated), proper_unit_for_byte_size(old_evacuated), - byte_size_in_proper_unit(old_available), proper_unit_for_byte_size(old_available)); - - size_t regular_promotion = promotion_reserve - consumed_by_advance_promotion; - size_t excess = - old_available - (old_evacuation_reserve + promotion_reserve + old_bytes_loaned_for_young_evac + allocation_supplement); - log_info(gc, ergo)("Old available: " SIZE_FORMAT "%s is partitioned into old evacuation budget: " SIZE_FORMAT - "%s, aged region promotion budget: " SIZE_FORMAT - "%s, regular region promotion budget: " SIZE_FORMAT - "%s, loaned for young evacuation: " SIZE_FORMAT - "%s, loaned for young allocations: " SIZE_FORMAT - "%s, excess: " SIZE_FORMAT "%s", - byte_size_in_proper_unit(old_available), proper_unit_for_byte_size(old_available), - byte_size_in_proper_unit(old_evacuation_reserve), proper_unit_for_byte_size(old_evacuation_reserve), - byte_size_in_proper_unit(consumed_by_advance_promotion), - proper_unit_for_byte_size(consumed_by_advance_promotion), - byte_size_in_proper_unit(regular_promotion), proper_unit_for_byte_size(regular_promotion), - byte_size_in_proper_unit(old_bytes_loaned_for_young_evac), - proper_unit_for_byte_size(old_bytes_loaned_for_young_evac), - byte_size_in_proper_unit(allocation_supplement), proper_unit_for_byte_size(allocation_supplement), - byte_size_in_proper_unit(excess), proper_unit_for_byte_size(excess)); + loaned_regions += additional_regions_to_loan; + old_bytes_reserved_for_alloc_supplement += additional_regions_to_loan * region_size_bytes; + working_old_available -= additional_regions_to_loan * region_size_bytes; } - // else, not generational: no evacuation budget adjustments required + size_t allocation_supplement = old_bytes_reserved_for_alloc_supplement; + heap->set_alloc_supplement_reserve(allocation_supplement); + + // TODO: young_available, which feeds into alloc_budget_evac_and_update is lacking memory available within + // existing young-gen regions that were not selected for the collection set. Add this in and adjust the + // log message (where it says "empty-region allocation budget"). + + log_debug(gc)("Memory reserved for young evacuation: " SIZE_FORMAT "%s for evacuating " SIZE_FORMAT + "%s out of young available: " SIZE_FORMAT "%s", + byte_size_in_proper_unit(young_evacuated_reserve_used), + proper_unit_for_byte_size(young_evacuated_reserve_used), + byte_size_in_proper_unit(young_evacuated), proper_unit_for_byte_size(young_evacuated), + byte_size_in_proper_unit(young_available), proper_unit_for_byte_size(young_available)); + + log_debug(gc)("Memory reserved for old evacuation: " SIZE_FORMAT "%s for evacuating " SIZE_FORMAT + "%s out of old available: " SIZE_FORMAT "%s", + byte_size_in_proper_unit(old_evacuated), proper_unit_for_byte_size(old_evacuated), + byte_size_in_proper_unit(old_evacuated), proper_unit_for_byte_size(old_evacuated), + byte_size_in_proper_unit(old_available), proper_unit_for_byte_size(old_available)); + + size_t regular_promotion = promotion_reserve - consumed_by_advance_promotion; + size_t excess = + old_available - (old_evacuation_reserve + promotion_reserve + old_bytes_loaned_for_young_evac + allocation_supplement); + log_info(gc, ergo)("Old available: " SIZE_FORMAT "%s is partitioned into old evacuation budget: " SIZE_FORMAT + "%s, aged region promotion budget: " SIZE_FORMAT + "%s, regular region promotion budget: " SIZE_FORMAT + "%s, loaned for young evacuation: " SIZE_FORMAT + "%s, loaned for young allocations: " SIZE_FORMAT + "%s, excess: " SIZE_FORMAT "%s", + byte_size_in_proper_unit(old_available), proper_unit_for_byte_size(old_available), + byte_size_in_proper_unit(old_evacuation_reserve), proper_unit_for_byte_size(old_evacuation_reserve), + byte_size_in_proper_unit(consumed_by_advance_promotion), + proper_unit_for_byte_size(consumed_by_advance_promotion), + byte_size_in_proper_unit(regular_promotion), proper_unit_for_byte_size(regular_promotion), + byte_size_in_proper_unit(old_bytes_loaned_for_young_evac), + proper_unit_for_byte_size(old_bytes_loaned_for_young_evac), + byte_size_in_proper_unit(allocation_supplement), proper_unit_for_byte_size(allocation_supplement), + byte_size_in_proper_unit(excess), proper_unit_for_byte_size(excess)); } void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) { ShenandoahHeap* heap = ShenandoahHeap::heap(); ShenandoahCollectionSet* collection_set = heap->collection_set(); - size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); assert(!heap->is_full_gc_in_progress(), "Only for concurrent and degenerated GC"); assert(generation_mode() != OLD, "Only YOUNG and GLOBAL GC perform evacuations"); @@ -791,40 +779,40 @@ void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) { ShenandoahFinalMarkUpdateRegionStateClosure old_cl(nullptr); heap->old_generation()->parallel_heap_region_iterate(&old_cl); } - - heap->assert_pinned_region_status(); } { - size_t consumed_by_advance_promotion; - bool* preselected_regions = nullptr; + ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::choose_cset : + ShenandoahPhaseTimings::degen_gc_choose_cset); + + collection_set->clear(); + ShenandoahHeapLocker locker(heap->lock()); if (heap->mode()->is_generational()) { - preselected_regions = (bool*) alloca(heap->num_regions() * sizeof(bool)); + size_t consumed_by_advance_promotion; + bool* preselected_regions = (bool*) alloca(heap->num_regions() * sizeof(bool)); for (unsigned int i = 0; i < heap->num_regions(); i++) { preselected_regions[i] = false; } - } - - ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::choose_cset : - ShenandoahPhaseTimings::degen_gc_choose_cset); - ShenandoahHeapLocker locker(heap->lock()); - collection_set->clear(); - // TODO: young_available can include available (between top() and end()) within each young region that is not - // part of the collection set. Making this memory available to the young_evacuation_reserve allows a larger - // young collection set to be chosen when available memory is under extreme pressure. Implementing this "improvement" - // is tricky, because the incremental construction of the collection set actually changes the amount of memory - // available to hold evacuated young-gen objects. As currently implemented, the memory that is available within - // non-empty regions that are not selected as part of the collection set can be allocated by the mutator while - // GC is evacuating and updating references. - - // Budgeting parameters to compute_evacuation_budgets are passed by reference. - compute_evacuation_budgets(heap, preselected_regions, collection_set, consumed_by_advance_promotion); - _heuristics->choose_collection_set(collection_set, heap->old_heuristics()); - if (!collection_set->is_empty()) { - adjust_evacuation_budgets(heap, collection_set, consumed_by_advance_promotion); + // TODO: young_available can include available (between top() and end()) within each young region that is not + // part of the collection set. Making this memory available to the young_evacuation_reserve allows a larger + // young collection set to be chosen when available memory is under extreme pressure. Implementing this "improvement" + // is tricky, because the incremental construction of the collection set actually changes the amount of memory + // available to hold evacuated young-gen objects. As currently implemented, the memory that is available within + // non-empty regions that are not selected as part of the collection set can be allocated by the mutator while + // GC is evacuating and updating references. + + // Budgeting parameters to compute_evacuation_budgets are passed by reference. + compute_evacuation_budgets(heap, preselected_regions, collection_set, consumed_by_advance_promotion); + + _heuristics->choose_collection_set(collection_set, heap->old_heuristics()); + if (!collection_set->is_empty()) { + // only make use of evacuation budgets when we are evacuating + adjust_evacuation_budgets(heap, collection_set, consumed_by_advance_promotion); + } + } else { + _heuristics->choose_collection_set(collection_set, heap->old_heuristics()); } - // otherwise, this is an abbreviated cycle and we make no use of evacuation budgets. } { From 264f9c2343244bdbb6f6e6d1d290f18495409da4 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Fri, 18 Nov 2022 23:22:03 +0000 Subject: [PATCH 156/254] Load balance remset scan Reviewed-by: wkemper --- .../share/gc/shenandoah/shenandoahHeap.cpp | 2 +- .../shenandoah/shenandoahScanRemembered.cpp | 130 +++++++++++++----- .../shenandoah/shenandoahScanRemembered.hpp | 68 ++++++--- .../shenandoahScanRemembered.inline.hpp | 18 +-- 4 files changed, 155 insertions(+), 63 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index da5488e57c7e9..7c5fcd6bdde88 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -1132,7 +1132,7 @@ HeapWord* ShenandoahHeap::allocate_new_plab(size_t min_size, } // is_promotion is true iff this allocation is known for sure to hold the result of young-gen evacuation -// to old-gen. plab allocates arre not known as such, since they may hold old-gen evacuations. +// to old-gen. plab allocates are not known as such, since they may hold old-gen evacuations. HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req, bool is_promotion) { intptr_t pacer_epoch = 0; bool in_new_region = false; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp index 837e6c8f337d3..6d2d4c754656e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp @@ -109,28 +109,37 @@ void ShenandoahScanRememberedTask::do_work(uint worker_id) { } } -size_t ShenandoahRegionChunkIterator::calc_group_size() { - // The group size s calculated from the number of regions. Every group except the last processes the same number of chunks. - // The last group processes however many chunks are required to finish the total scanning effort. The chunk sizes are +size_t ShenandoahRegionChunkIterator::calc_regular_group_size() { + // The group size is calculated from the number of regions. Suppose the entire heap size is N. The first group processes + // N/2 of total heap size. The second group processes N/4 of total heap size. The third group processes N/2 of total heap + // size, and so on. Note that N/2 + N/4 + N/8 + N/16 + ... sums to N if expanded to infinite terms. + // + // The normal group size is the number of regions / 2. + // + // In the case that the region_size_words is greater than _maximum_chunk_size_words, the first group_size is + // larger than the normal group size because each chunk in the group will be smaller than the region size. + // + // The last group also has more than the normal entries because it finishes the total scanning effort. The chunk sizes are // different for each group. The intention is that the first group processes roughly half of the heap, the second processes // a quarter of the remaining heap, the third processes an eight of what remains and so on. The smallest chunk size - // is represented by _smallest_chunk_size. We do not divide work any smaller than this. + // is represented by _smallest_chunk_size_words. We do not divide work any smaller than this. // - // Note that N/2 + N/4 + N/8 + N/16 + ... sums to N if expanded to infinite terms. - return _heap->num_regions() / 2; + + size_t group_size = _heap->num_regions() / 2; + return group_size; } -size_t ShenandoahRegionChunkIterator::calc_first_group_chunk_size() { - size_t words_in_region = ShenandoahHeapRegion::region_size_words(); - return words_in_region; +size_t ShenandoahRegionChunkIterator::calc_first_group_chunk_size_b4_rebalance() { + size_t words_in_first_chunk = ShenandoahHeapRegion::region_size_words(); + return words_in_first_chunk; } size_t ShenandoahRegionChunkIterator::calc_num_groups() { size_t total_heap_size = _heap->num_regions() * ShenandoahHeapRegion::region_size_words(); size_t num_groups = 0; size_t cumulative_group_span = 0; - size_t current_group_span = _first_group_chunk_size * _group_size; - size_t smallest_group_span = _smallest_chunk_size * _group_size; + size_t current_group_span = _first_group_chunk_size_b4_rebalance * _regular_group_size; + size_t smallest_group_span = _smallest_chunk_size_words * _regular_group_size; while ((num_groups < _maximum_groups) && (cumulative_group_span + current_group_span <= total_heap_size)) { num_groups++; cumulative_group_span += current_group_span; @@ -144,7 +153,7 @@ size_t ShenandoahRegionChunkIterator::calc_num_groups() { // num_groups <= _maximum_groups // cumulative_group_span is the memory spanned by num_groups // current_group_span is the span of the last fully populated group (assuming loop iterates at least once) - // each of num_groups is fully populated with _group_size chunks in each + // each of num_groups is fully populated with _regular_group_size chunks in each // Non post conditions: // cumulative_group_span may be less than total_heap size for one or more of the folowing reasons // a) The number of regions remaining to be spanned is smaller than a complete group, or @@ -159,7 +168,7 @@ size_t ShenandoahRegionChunkIterator::calc_num_groups() { // minimum chunk size. // Any remaining regions will be treated as if they are part of the most recently created group. This group will - // have more than _group_size chunks within it. + // have more than _regular_group_size chunks within it. } return num_groups; } @@ -168,29 +177,46 @@ size_t ShenandoahRegionChunkIterator::calc_total_chunks() { size_t region_size_words = ShenandoahHeapRegion::region_size_words(); size_t unspanned_heap_size = _heap->num_regions() * region_size_words; size_t num_chunks = 0; - size_t spanned_groups = 0; size_t cumulative_group_span = 0; - size_t current_group_span = _first_group_chunk_size * _group_size; - size_t smallest_group_span = _smallest_chunk_size * _group_size; + size_t current_group_span = _first_group_chunk_size_b4_rebalance * _regular_group_size; + size_t smallest_group_span = _smallest_chunk_size_words * _regular_group_size; + + // The first group gets special handling because the first chunk size can be no larger than _largest_chunk_size_words + if (region_size_words > _maximum_chunk_size_words) { + // In the case that we shrink the first group's chunk size, certain other groups will also be subsumed within the first group + size_t effective_chunk_size = _first_group_chunk_size_b4_rebalance; + while (effective_chunk_size >= _maximum_chunk_size_words) { + num_chunks += current_group_span / _maximum_chunk_size_words; + unspanned_heap_size -= current_group_span; + effective_chunk_size /= 2; + current_group_span /= 2; + } + } else { + num_chunks = _regular_group_size; + unspanned_heap_size -= current_group_span; + current_group_span /= 2; + } + size_t spanned_groups = 1; while (unspanned_heap_size > 0) { if (current_group_span <= unspanned_heap_size) { unspanned_heap_size -= current_group_span; - num_chunks += _group_size; + num_chunks += _regular_group_size; spanned_groups++; // _num_groups is the number of groups required to span the configured heap size. We are not allowed // to change the number of groups. The last group is responsible for spanning all chunks not spanned // by previously processed groups. if (spanned_groups >= _num_groups) { - // The last group has more than _group_size entries. - size_t chunk_span = current_group_span / _group_size; + // The last group has more than _regular_group_size entries. + size_t chunk_span = current_group_span / _regular_group_size; size_t extra_chunks = unspanned_heap_size / chunk_span; assert (extra_chunks * chunk_span == unspanned_heap_size, "Chunks must precisely span regions"); num_chunks += extra_chunks; return num_chunks; } else if (current_group_span <= smallest_group_span) { - // We cannot introduce new groups because we've reached the lower bound on group size - size_t chunk_span = _smallest_chunk_size; + // We cannot introduce new groups because we've reached the lower bound on group size. So this last + // group may hold extra chunks. + size_t chunk_span = _smallest_chunk_size_words; size_t extra_chunks = unspanned_heap_size / chunk_span; assert (extra_chunks * chunk_span == unspanned_heap_size, "Chunks must precisely span regions"); num_chunks += extra_chunks; @@ -199,8 +225,8 @@ size_t ShenandoahRegionChunkIterator::calc_total_chunks() { current_group_span /= 2; } } else { - // The last group has fewer than _group_size entries. - size_t chunk_span = current_group_span / _group_size; + // This last group has fewer than _regular_group_size entries. + size_t chunk_span = current_group_span / _regular_group_size; size_t last_group_size = unspanned_heap_size / chunk_span; assert (last_group_size * chunk_span == unspanned_heap_size, "Chunks must precisely span regions"); num_chunks += last_group_size; @@ -217,30 +243,68 @@ ShenandoahRegionChunkIterator::ShenandoahRegionChunkIterator(size_t worker_count ShenandoahRegionChunkIterator::ShenandoahRegionChunkIterator(ShenandoahHeap* heap, size_t worker_count) : _heap(heap), - _group_size(calc_group_size()), - _first_group_chunk_size(calc_first_group_chunk_size()), + _regular_group_size(calc_regular_group_size()), + _first_group_chunk_size_b4_rebalance(calc_first_group_chunk_size_b4_rebalance()), _num_groups(calc_num_groups()), _total_chunks(calc_total_chunks()), _index(0) { - assert(_smallest_chunk_size == - CardTable::card_size_in_words() * ShenandoahCardCluster::CardsPerCluster, - "_smallest_chunk_size is not valid"); + assert(_smallest_chunk_size_words == _clusters_in_smallest_chunk * CardTable::card_size_in_words() + * ShenandoahCardCluster::CardsPerCluster, "_smallest_chunk_size is not valid"); + assert(_num_groups <= _maximum_groups, + "The number of remembered set scanning groups must be less than or equal to maximum groups"); + assert(_smallest_chunk_size_words << (_maximum_groups - 1) == _maximum_chunk_size_words, + "Maximum number of groups needs to span maximum chunk size to smallest chunk size"); size_t words_in_region = ShenandoahHeapRegion::region_size_words(); - size_t group_span = _first_group_chunk_size * _group_size; - _region_index[0] = 0; _group_offset[0] = 0; + if (words_in_region > _maximum_chunk_size_words) { + // In the case that we shrink the first group's chunk size, certain other groups will also be subsumed within the first group + size_t num_chunks = 0; + size_t effective_chunk_size = _first_group_chunk_size_b4_rebalance; + size_t current_group_span = effective_chunk_size * _regular_group_size; + while (effective_chunk_size >= _maximum_chunk_size_words) { + num_chunks += current_group_span / _maximum_chunk_size_words; + effective_chunk_size /= 2; + current_group_span /= 2; + } + _group_entries[0] = num_chunks; + _group_chunk_size[0] = _maximum_chunk_size_words; + } else { + _group_entries[0] = _regular_group_size; + _group_chunk_size[0] = _first_group_chunk_size_b4_rebalance; + } + + size_t previous_group_span = _group_entries[0] * _group_chunk_size[0]; for (size_t i = 1; i < _num_groups; i++) { - _region_index[i] = _region_index[i-1] + (_group_offset[i-1] + group_span) / words_in_region; - _group_offset[i] = (_group_offset[i-1] + group_span) % words_in_region; - group_span /= 2; + size_t previous_group_entries = (i == 1)? _group_entries[0]: (_group_entries[i-1] - _group_entries[i-2]); + _group_chunk_size[i] = _group_chunk_size[i-1] / 2; + size_t chunks_in_group = _regular_group_size; + size_t this_group_span = _group_chunk_size[i] * chunks_in_group; + size_t total_span_of_groups = previous_group_span + this_group_span; + _region_index[i] = previous_group_span / words_in_region; + _group_offset[i] = previous_group_span % words_in_region; + _group_entries[i] = _group_entries[i-1] + _regular_group_size; + previous_group_span = total_span_of_groups; } + if (_group_entries[_num_groups-1] < _total_chunks) { + assert((_total_chunks - _group_entries[_num_groups-1]) * _group_chunk_size[_num_groups-1] + previous_group_span == + heap->num_regions() * words_in_region, "Total region chunks (" SIZE_FORMAT + ") do not span total heap regions (" SIZE_FORMAT ")", _total_chunks, _heap->num_regions()); + previous_group_span += (_total_chunks - _group_entries[_num_groups-1]) * _group_chunk_size[_num_groups-1]; + _group_entries[_num_groups-1] = _total_chunks; + } + assert(previous_group_span == heap->num_regions() * words_in_region, "Total region chunks (" SIZE_FORMAT + ") do not span total heap regions (" SIZE_FORMAT "): " SIZE_FORMAT " does not equal " SIZE_FORMAT, + _total_chunks, _heap->num_regions(), previous_group_span, heap->num_regions() * words_in_region); + // Not necessary, but keeps things tidy for (size_t i = _num_groups; i < _maximum_groups; i++) { _region_index[i] = 0; _group_offset[i] = 0; + _group_entries[i] = _group_entries[i-1]; + _group_chunk_size[i] = 0; } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp index d7657bffba3a9..053bb08d26261 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp @@ -1006,32 +1006,59 @@ struct ShenandoahRegionChunk { size_t _chunk_size; // HeapWordSize qty }; +// ShenandoahRegionChunkIterator divides the total remembered set scanning effort into assignments (ShenandoahRegionChunks) +// that are assigned one at a time to worker threads. Note that the effort required to scan a range of memory is not +// necessarily a linear function of the size of the range. Some memory ranges hold only a small number of live objects. +// Some ranges hold primarily primitive (non-pointer data). We start with larger assignment sizes because larger assignments +// can be processed with less coordination effort. We expect that the GC worker threads that receive more difficult assignments +// will work longer on those assignments. Meanwhile, other worker will threads repeatedly accept and complete multiple +// easier assignments. As the total amount of work remaining to be completed decreases, we decrease the size of assignments +// given to individual threads. This reduces the likelihood that significant imbalance between worker thread assignments +// will be introduced when there is less meaningful work to be performed by the remaining worker threads while they wait for +// worker threads with difficult assignments to finish. + class ShenandoahRegionChunkIterator : public StackObj { private: - // smallest_chunk_size is 64 words per card * - // ShenandoahCardCluster::CardsPerCluster. + // The largest chunk size is 4 MiB, measured in words. Otherwise, remembered set scanning may become too unbalanced. + // If the largest chunk size is too small, there is too much overhead sifting out assignments to individual worker threads. + static const size_t _maximum_chunk_size_words = (4 * 1024 * 1024) / HeapWordSize; + + static const size_t _clusters_in_smallest_chunk = 4; + static const size_t _assumed_words_in_card = 64; + + // smallest_chunk_size is 4 clusters (i.e. 128 KiB). Note that there are 64 words per card and there are 64 cards per + // cluster. Each cluster spans 128 KiB. // This is computed from CardTable::card_size_in_words() * // ShenandoahCardCluster::CardsPerCluster; // We can't perform this computation here, because of encapsulation and initialization constraints. We paste // the magic number here, and assert that this number matches the intended computation in constructor. - static const size_t _smallest_chunk_size = 64 * ShenandoahCardCluster::CardsPerCluster; + static const size_t _smallest_chunk_size_words = (_clusters_in_smallest_chunk * _assumed_words_in_card * + ShenandoahCardCluster::CardsPerCluster); // The total remembered set scanning effort is divided into chunks of work that are assigned to individual worker tasks. - // The chunks of assigned work are divided into groups, where the size of each group (_group_size) is 4 * the number of - // worker tasks. All of the assignments within a group represent the same amount of memory to be scanned. Each of the - // assignments within the first group are of size _first_group_chunk_size (typically the ShenandoahHeapRegion size, but - // possibly smaller. Each of the assignments within each subsequent group are half the size of the assignments in the - // preceding group. The last group may be larger than the others. Because no group is allowed to have smaller assignments - // than _smallest_chunk_size, which is 32 KB. + // The chunks of assigned work are divided into groups, where the size of the typical group (_regular_group_size) is half the + // total number of regions. The first group may be larger than + // _regular_group_size in the case that the first group's chunk + // size is less than the region size. The last group may be larger + // than _regular_group_size because no group is allowed to + // have smaller assignments than _smallest_chunk_size, which is 128 KB. // Under normal circumstances, no configuration needs more than _maximum_groups (default value of 16). + // The first group "effectively" processes chunks of size 1 MiB (or smaller for smaller region sizes). + // The last group processes chunks of size 128 KiB. There are four groups total. - static const size_t _maximum_groups = 16; + // group[0] is 4 MiB chunk size (_maximum_chunk_size_words) + // group[1] is 2 MiB chunk size + // group[2] is 1 MiB chunk size + // group[3] is 512 KiB chunk size + // group[4] is 256 KiB chunk size + // group[5] is 128 Kib shunk size (_smallest_chunk_size_words = 4 * 64 * 64 + static const size_t _maximum_groups = 6; const ShenandoahHeap* _heap; - const size_t _group_size; // Number of chunks in each group, equals worker_threads * 8 - const size_t _first_group_chunk_size; + const size_t _regular_group_size; // Number of chunks in each group + const size_t _first_group_chunk_size_b4_rebalance; const size_t _num_groups; // Number of groups in this configuration const size_t _total_chunks; @@ -1039,23 +1066,24 @@ class ShenandoahRegionChunkIterator : public StackObj { volatile size_t _index; shenandoah_padding(1); - size_t _region_index[_maximum_groups]; - size_t _group_offset[_maximum_groups]; - + size_t _region_index[_maximum_groups]; // The region index for the first region spanned by this group + size_t _group_offset[_maximum_groups]; // The offset at which group begins within first region spanned by this group + size_t _group_chunk_size[_maximum_groups]; // The size of each chunk within this group + size_t _group_entries[_maximum_groups]; // Total chunks spanned by this group and the ones before it. // No implicit copying: iterators should be passed by reference to capture the state NONCOPYABLE(ShenandoahRegionChunkIterator); // Makes use of _heap. - size_t calc_group_size(); + size_t calc_regular_group_size(); - // Makes use of _group_size, which must be initialized before call. - size_t calc_first_group_chunk_size(); + // Makes use of _regular_group_size, which must be initialized before call. + size_t calc_first_group_chunk_size_b4_rebalance(); - // Makes use of _group_size and _first_group_chunk_size, both of which must be initialized before call. + // Makes use of _regular_group_size and _first_group_chunk_size_b4_rebalance, both of which must be initialized before call. size_t calc_num_groups(); - // Makes use of _group_size, _first_group_chunk_size, which must be initialized before call. + // Makes use of _regular_group_size, _first_group_chunk_size_b4_rebalance, which must be initialized before call. size_t calc_total_chunks(); public: diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp index a029314db798d..936882e40476c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp @@ -678,7 +678,6 @@ ShenandoahScanRemembered::process_humongous_clusters(ShenandoahHe size_t first_card_index = first_cluster * ShenandoahCardCluster::CardsPerCluster; HeapWord* first_cluster_addr = _rs->addr_for_card_index(first_card_index); size_t spanned_words = count * ShenandoahCardCluster::CardsPerCluster * CardTable::card_size_in_words(); - start_region->oop_iterate_humongous_slice(cl, true, first_cluster_addr, spanned_words, write_table, is_concurrent); } @@ -815,27 +814,28 @@ inline bool ShenandoahRegionChunkIterator::next(struct ShenandoahRegionChunk *as // convert to zero-based indexing new_index--; - size_t group_no = new_index / _group_size; - if (group_no + 1 > _num_groups) { - group_no = _num_groups - 1; - } + size_t group_no; + for (group_no = 0; new_index >= _group_entries[group_no]; group_no++) + ; + + assert(group_no < _num_groups, "Cannot have group no greater or equal to _num_groups"); // All size computations measured in HeapWord size_t region_size_words = ShenandoahHeapRegion::region_size_words(); size_t group_region_index = _region_index[group_no]; size_t group_region_offset = _group_offset[group_no]; - size_t index_within_group = new_index - (group_no * _group_size); - size_t group_chunk_size = _first_group_chunk_size >> group_no; + size_t index_within_group = (group_no == 0)? new_index: new_index - _group_entries[group_no - 1]; + size_t group_chunk_size = _group_chunk_size[group_no]; size_t offset_of_this_chunk = group_region_offset + index_within_group * group_chunk_size; size_t regions_spanned_by_chunk_offset = offset_of_this_chunk / region_size_words; - size_t region_index = group_region_index + regions_spanned_by_chunk_offset; size_t offset_within_region = offset_of_this_chunk % region_size_words; + size_t region_index = group_region_index + regions_spanned_by_chunk_offset; + assignment->_r = _heap->get_region(region_index); assignment->_chunk_offset = offset_within_region; assignment->_chunk_size = group_chunk_size; - return true; } From b594e541311eae053aea0664be3f2453c62b95f4 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Sat, 19 Nov 2022 00:12:36 +0000 Subject: [PATCH 157/254] Various build fixes Reviewed-by: kdnilsen --- .../shenandoahBarrierSetAssembler_riscv.cpp | 4 ++-- .../gc/shenandoah/shenandoahEvacTracker.cpp | 2 ++ .../share/gc/shenandoah/shenandoahHeap.cpp | 8 ++++---- .../shenandoahHeapRegionCounters.cpp | 6 +++--- .../test_shenandoahOldHeuristic.cpp | 20 ++++++++++++++----- 5 files changed, 26 insertions(+), 14 deletions(-) diff --git a/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.cpp b/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.cpp index bb20c02629b45..07aa390c5399e 100644 --- a/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.cpp +++ b/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.cpp @@ -64,7 +64,7 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec __ andi(t0, t0, ShenandoahHeap::HAS_FORWARDED); __ beqz(t0, done); } else { - __ andi(t0, t0, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING); + __ andi(t0, t0, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING); __ beqz(t0, done); } @@ -642,7 +642,7 @@ void ShenandoahBarrierSetAssembler::generate_c1_pre_barrier_runtime_stub(StubAss // Is marking still active? Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); __ lb(tmp, gc_state); - __ andi(tmp, tmp, ShenandoahHeap::MARKING); + __ andi(tmp, tmp, ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING); __ beqz(tmp, done); // Can we store original value in the thread's buffer? diff --git a/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.cpp b/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.cpp index bd563e4d49861..87188b75358a3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.cpp @@ -21,7 +21,9 @@ * questions. * */ +#include "precompiled.hpp" +#include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahEvacTracker.hpp" #include "gc/shenandoah/shenandoahThreadLocalData.hpp" #include "gc/shenandoah/shenandoahRootProcessor.hpp" diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 7c5fcd6bdde88..d65eb1093b547 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -832,9 +832,9 @@ void ShenandoahHeap::report_promotion_failure(Thread* thread, size_t size) { // We squelch excessive reports to reduce noise in logs. Squelch enforcement is not "perfect" because // this same code can be in-lined in multiple contexts, and each context will have its own copy of the static // last_report_epoch and this_epoch_report_count variables. - const uint MaxReportsPerEpoch = 4; - static uint last_report_epoch = 0; - static uint epoch_report_count = 0; + const size_t MaxReportsPerEpoch = 4; + static size_t last_report_epoch = 0; + static size_t epoch_report_count = 0; size_t promotion_reserve; size_t promotion_expended; @@ -857,7 +857,7 @@ void ShenandoahHeap::report_promotion_failure(Thread* thread, size_t size) { size, plab == nullptr? "no": "yes", words_remaining, promote_enabled, promotion_reserve, promotion_expended); if ((gc_id == last_report_epoch) && (epoch_report_count >= MaxReportsPerEpoch)) { - log_info(gc, ergo)("Squelching additional promotion failure reports for epoch %d", last_report_epoch); + log_info(gc, ergo)("Squelching additional promotion failure reports for epoch " SIZE_FORMAT, last_report_epoch); } else if (gc_id != last_report_epoch) { last_report_epoch = gc_id;; epoch_report_count = 1; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp index f0f6c6b3b6c14..d36f5ce8dcf12 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp @@ -91,13 +91,13 @@ void ShenandoahHeapRegionCounters::write_snapshot(PerfLongVariable** regions, ResourceMark rm; LogStream ls(lt); - ls.print_cr("%li %li %lu %lu %lu", + ls.print_cr(JLONG_FORMAT " " JLONG_FORMAT " " SIZE_FORMAT " " SIZE_FORMAT " " SIZE_FORMAT, ts->get_value(), status->get_value(), num_regions, region_size, protocol_version); if (num_regions > 0) { - ls.print("%li", regions[0]->get_value()); + ls.print(JLONG_FORMAT, regions[0]->get_value()); } for (uint i = 1; i < num_regions; ++i) { - ls.print(" %li", regions[i]->get_value()); + ls.print(" " JLONG_FORMAT, regions[i]->get_value()); } ls.cr(); } diff --git a/test/hotspot/gtest/gc/shenandoah/test_shenandoahOldHeuristic.cpp b/test/hotspot/gtest/gc/shenandoah/test_shenandoahOldHeuristic.cpp index 850a8b63b4d96..8ac2926bb5f78 100644 --- a/test/hotspot/gtest/gc/shenandoah/test_shenandoahOldHeuristic.cpp +++ b/test/hotspot/gtest/gc/shenandoah/test_shenandoahOldHeuristic.cpp @@ -17,13 +17,19 @@ // collector in ways that are not compatible with a normal collection run. // If these tests take longer than the minimum time between gc intervals - // or, more likely, if you have them paused in a debugger longer than this -// interval - you can expect trouble. - +// interval - you can expect trouble. These tests will also not run in a build +// with asserts enabled because they use APIs that expect to run on a safepoint. +#ifdef ASSERT +#define SKIP_IF_NOT_SHENANDOAH() \ + tty->print_cr("skipped (debug build)" ); \ + return; +#else #define SKIP_IF_NOT_SHENANDOAH() \ if (!UseShenandoahGC) { \ tty->print_cr("skipped"); \ return; \ } +#endif class ShenandoahResetRegions : public ShenandoahHeapRegionClosure { public: @@ -45,9 +51,13 @@ class ShenandoahOldHeuristicTest : public ::testing::Test { ShenandoahCollectionSet* _collection_set; ShenandoahOldHeuristicTest() - : _heap(ShenandoahHeap::heap()), - _heuristics(_heap->old_heuristics()), - _collection_set(_heap->collection_set()) { + : _heap(nullptr), + _heuristics(nullptr), + _collection_set(nullptr) { + SKIP_IF_NOT_SHENANDOAH(); + _heap = ShenandoahHeap::heap(); + _heuristics = _heap->old_heuristics(); + _collection_set = _heap->collection_set(); ShenandoahHeapLocker locker(_heap->lock()); ShenandoahResetRegions reset; _heap->heap_region_iterate(&reset); From 25469283fbe14e85adeaf0e3a21d40faea5f7288 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Wed, 7 Dec 2022 22:53:11 +0000 Subject: [PATCH 158/254] Enforce max regions Reviewed-by: wkemper, ysr --- .../share/gc/shenandoah/shenandoahFreeSet.cpp | 69 ++++++++++++++----- .../share/gc/shenandoah/shenandoahFullGC.cpp | 10 ++- .../gc/shenandoah/shenandoahGeneration.cpp | 15 +++- .../gc/shenandoah/shenandoahGeneration.hpp | 12 +++- .../shenandoah/shenandoahGlobalGeneration.cpp | 5 ++ .../shenandoah/shenandoahGlobalGeneration.hpp | 7 +- .../share/gc/shenandoah/shenandoahHeap.cpp | 22 +++++- .../share/gc/shenandoah/shenandoahHeap.hpp | 5 +- .../gc/shenandoah/shenandoahHeapRegion.cpp | 68 +++++++++++++----- .../gc/shenandoah/shenandoahVerifier.cpp | 24 ++++++- 10 files changed, 185 insertions(+), 52 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index 6fd770697a11e..deee5fed0e0e8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -97,14 +97,37 @@ HeapWord* ShenandoahFreeSet::allocate_single(ShenandoahAllocRequest& req, bool& // Overwrite with non-zero (non-NULL) values only if necessary for allocation bookkeeping. + bool allow_new_region = true; + switch (req.affiliation()) { + case ShenandoahRegionAffiliation::OLD_GENERATION: + // Note: unsigned result from adjusted_unaffiliated_regions() will never be less than zero, but it may equal zero. + if (_heap->old_generation()->adjusted_unaffiliated_regions() <= 0) { + allow_new_region = false; + } + break; + + case ShenandoahRegionAffiliation::YOUNG_GENERATION: + // Note: unsigned result from adjusted_unaffiliated_regions() will never be less than zero, but it may equal zero. + if (_heap->young_generation()->adjusted_unaffiliated_regions() <= 0) { + allow_new_region = false; + } + break; + + case ShenandoahRegionAffiliation::FREE: + default: + ShouldNotReachHere(); + break; + } + switch (req.type()) { case ShenandoahAllocRequest::_alloc_tlab: case ShenandoahAllocRequest::_alloc_shared: { // Try to allocate in the mutator view for (size_t idx = _mutator_leftmost; idx <= _mutator_rightmost; idx++) { - if (is_mutator_free(idx)) { + ShenandoahHeapRegion* r = _heap->get_region(idx); + if (is_mutator_free(idx) && (allow_new_region || r->affiliation() != ShenandoahRegionAffiliation::FREE)) { // try_allocate_in() increases used if the allocation is successful. - HeapWord* result = try_allocate_in(_heap->get_region(idx), req, in_new_region); + HeapWord* result = try_allocate_in(r, req, in_new_region); if (result != NULL) { return result; } @@ -127,10 +150,12 @@ HeapWord* ShenandoahFreeSet::allocate_single(ShenandoahAllocRequest& req, bool& if (result != NULL) { return result; } - // Then try a free region that is dedicated to GC allocations. - result = allocate_with_affiliation(FREE, req, in_new_region); - if (result != NULL) { - return result; + if (allow_new_region) { + // Then try a free region that is dedicated to GC allocations. + result = allocate_with_affiliation(FREE, req, in_new_region); + if (result != NULL) { + return result; + } } // No dice. Can we borrow space from mutator view? @@ -138,16 +163,18 @@ HeapWord* ShenandoahFreeSet::allocate_single(ShenandoahAllocRequest& req, bool& return NULL; } - // Try to steal an empty region from the mutator view. - for (size_t c = _mutator_rightmost + 1; c > _mutator_leftmost; c--) { - size_t idx = c - 1; - if (is_mutator_free(idx)) { - ShenandoahHeapRegion* r = _heap->get_region(idx); - if (can_allocate_from(r)) { - flip_to_gc(r); - HeapWord *result = try_allocate_in(r, req, in_new_region); - if (result != NULL) { - return result; + if (allow_new_region) { + // Try to steal an empty region from the mutator view. + for (size_t c = _mutator_rightmost + 1; c > _mutator_leftmost; c--) { + size_t idx = c - 1; + if (is_mutator_free(idx)) { + ShenandoahHeapRegion* r = _heap->get_region(idx); + if (can_allocate_from(r)) { + flip_to_gc(r); + HeapWord *result = try_allocate_in(r, req, in_new_region); + if (result != NULL) { + return result; + } } } } @@ -176,9 +203,7 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah if (r->affiliation() == ShenandoahRegionAffiliation::FREE) { ShenandoahMarkingContext* const ctx = _heap->complete_marking_context(); - r->set_affiliation(req.affiliation()); - if (r->is_old()) { // Any OLD region allocated during concurrent coalesce-and-fill does not need to be coalesced and filled because // all objects allocated within this region are above TAMS (and thus are implicitly marked). In case this is an @@ -243,6 +268,9 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah if (size >= req.min_size()) { result = r->allocate(size, req); assert (result != NULL, "Allocation must succeed: free " SIZE_FORMAT ", actual " SIZE_FORMAT, free, size); + } else { + log_info(gc, ergo)("Failed to shrink TLAB or GCLAB request (" SIZE_FORMAT ") in region " SIZE_FORMAT " to " SIZE_FORMAT + " because min_size() is " SIZE_FORMAT, req.size(), r->index(), size, req.min_size()); } } } else if (req.is_lab_alloc() && req.type() == ShenandoahAllocRequest::_alloc_plab) { @@ -372,8 +400,11 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req) { size_t words_size = req.size(); size_t num = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize); + assert(req.affiliation() == ShenandoahRegionAffiliation::YOUNG_GENERATION, "Humongous regions always allocated in YOUNG"); + size_t avail_young_regions = _heap->young_generation()->adjusted_unaffiliated_regions(); + // No regions left to satisfy allocation, bye. - if (num > mutator_count()) { + if (num > mutator_count() || (num > avail_young_regions)) { return NULL; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index c8ff412c5d1ea..444614bbbe17f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -195,9 +195,9 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) { heap->set_gc_generation(heap->global_generation()); if (heap->mode()->is_generational()) { - // There will be no concurrent allocations during full GC so reset these coordination variables. - heap->young_generation()->unadjust_available(); - heap->old_generation()->unadjust_available(); + // Defer unadjust_available() invocations until after Full GC finishes its efforts because Full GC makes use + // of young-gen memory that may have been loaned from old-gen. + // No need to old_gen->increase_used(). That was done when plabs were allocated, accounting for both old evacs and promotions. heap->set_alloc_supplement_reserve(0); @@ -353,6 +353,10 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) { } } + // Having reclaimed all dead memory, it is now safe to restore capacities to original values. + heap->young_generation()->unadjust_available(); + heap->old_generation()->unadjust_available(); + if (VerifyAfterGC) { Universe::verify(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index 5f906b83cfc2a..9643485b8cdb9 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -914,12 +914,14 @@ void ShenandoahGeneration::scan_remembered_set(bool is_concurrent) { heap->workers()->run_task(&task); } -void ShenandoahGeneration::increment_affiliated_region_count() { +size_t ShenandoahGeneration::increment_affiliated_region_count() { _affiliated_region_count++; + return _affiliated_region_count; } -void ShenandoahGeneration::decrement_affiliated_region_count() { +size_t ShenandoahGeneration::decrement_affiliated_region_count() { _affiliated_region_count--; + return _affiliated_region_count; } void ShenandoahGeneration::clear_used() { @@ -977,6 +979,15 @@ size_t ShenandoahGeneration::adjusted_available() const { return in_use > capacity ? 0 : capacity - in_use; } +size_t ShenandoahGeneration::adjusted_capacity() const { + return _adjusted_capacity; +} + +size_t ShenandoahGeneration::adjusted_unaffiliated_regions() const { + assert(adjusted_capacity() > used_regions_size(), "adjusted_unaffiliated_regions() cannot return negative"); + return (adjusted_capacity() - used_regions_size()) / ShenandoahHeapRegion::region_size_bytes(); +} + void ShenandoahGeneration::record_success_concurrent(bool abbreviated) { heuristics()->record_success_concurrent(abbreviated); ShenandoahHeap::heap()->shenandoah_policy()->record_success_concurrent(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index e59ac8dee09f0..d3941a37c0af4 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -95,6 +95,11 @@ class ShenandoahGeneration : public CHeapObj { // set aside within each generation to hold the results of evacuation, but not promotion, into that region. Promotions // into old-gen are bounded by adjusted_available() whereas evacuations into old-gen are pre-committed. virtual size_t adjusted_available() const; + virtual size_t adjusted_capacity() const; + + // This is the number of FREE regions that are eligible to be affiliated with this generation according to the current + // adjusted capacity. + virtual size_t adjusted_unaffiliated_regions() const; // Both of following return new value of available virtual size_t adjust_available(intptr_t adjustment); @@ -161,8 +166,11 @@ class ShenandoahGeneration : public CHeapObj { // Scan remembered set at start of concurrent young-gen marking. */ void scan_remembered_set(bool is_concurrent); - void increment_affiliated_region_count(); - void decrement_affiliated_region_count(); + // Return the updated value of affiliated_region_count + size_t increment_affiliated_region_count(); + + // Return the updated value of affiliated_region_count + size_t decrement_affiliated_region_count(); void clear_used(); void increase_used(size_t bytes); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp index fd576c224e2ce..8bda79d480582 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp @@ -40,6 +40,11 @@ size_t ShenandoahGlobalGeneration::max_capacity() const { return ShenandoahHeap::heap()->max_capacity(); } +size_t ShenandoahGlobalGeneration::used_regions() const { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + return heap->old_generation()->used_regions() + heap->young_generation()->used_regions(); +} + size_t ShenandoahGlobalGeneration::used_regions_size() const { return ShenandoahHeap::heap()->capacity(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp index 098aad6fa6ce0..fb5e043110e6b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp @@ -26,18 +26,21 @@ #define SHARE_VM_GC_SHENANDOAH_SHENANDOAHGLOBALGENERATION_HPP #include "gc/shenandoah/shenandoahGeneration.hpp" +#include "gc/shenandoah/shenandoahYoungGeneration.hpp" +#include "gc/shenandoah/shenandoahOldGeneration.hpp" // A "generation" that represents the whole heap. class ShenandoahGlobalGeneration : public ShenandoahGeneration { public: - ShenandoahGlobalGeneration(uint max_queues) - : ShenandoahGeneration(GLOBAL, max_queues, 0, 0) { } + ShenandoahGlobalGeneration(uint max_queues, size_t max_capacity, size_t soft_max_capacity) + : ShenandoahGeneration(GLOBAL, max_queues, max_capacity, soft_max_capacity) { } public: virtual const char* name() const override; virtual size_t max_capacity() const override; virtual size_t soft_max_capacity() const override; + virtual size_t used_regions() const override; virtual size_t used_regions_size() const override; virtual size_t used() const override; virtual size_t available() const override; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index d65eb1093b547..caa993b9600aa 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -455,7 +455,8 @@ void ShenandoahHeap::initialize_generations() { _young_generation = new ShenandoahYoungGeneration(_max_workers, max_capacity_new, soft_max_capacity_new); _old_generation = new ShenandoahOldGeneration(_max_workers, max_capacity_old, soft_max_capacity_old); - _global_generation = new ShenandoahGlobalGeneration(_max_workers); + _global_generation = new ShenandoahGlobalGeneration(_max_workers, max_capacity_new + max_capacity_old, + soft_max_capacity_new + soft_max_capacity_old); } void ShenandoahHeap::initialize_heuristics() { @@ -683,6 +684,8 @@ size_t ShenandoahHeap::young_generation_capacity(size_t capacity) { capacity = MIN2(MaxNewSize, capacity); } } + // capacity must be a multiple of ShenandoahHeapRegion::region_size_bytes() + capacity &= ~ShenandoahHeapRegion::region_size_bytes_mask(); } // else, make no adjustment to global capacity return capacity; @@ -1227,6 +1230,12 @@ HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req if (requested_bytes >= young_generation()->adjusted_available()) { // We know this is not a GCLAB. This must be a TLAB or a shared allocation. Reject the allocation request if // exceeds established capacity limits. + + // TODO: if ShenandoahElasticTLAB and req.is_lab_alloc(), we should endeavor to shrink the TLAB request + // in order to avoid allocation failure and degeneration of GC. + + log_info(gc, ergo)("Rejecting mutator alloc of " SIZE_FORMAT " because young available is: " SIZE_FORMAT, + requested_bytes, young_generation()->adjusted_available()); return nullptr; } } @@ -1479,7 +1488,16 @@ class ShenandoahGenerationalEvacuationTask : public WorkerTask { // We promote humongous_start regions along with their affiliated continuations during evacuation rather than // doing this work during a safepoint. We cannot put humongous regions into the collection set because that // triggers the load-reference barrier (LRB) to copy on reference fetch. - r->promote_humongous(); + if (r->promote_humongous() == 0) { + // We chose not to promote because old-gen is out of memory. Report and handle the promotion failure because + // this suggests need for expanding old-gen and/or performing collection of old-gen. + ShenandoahHeap* heap = ShenandoahHeap::heap(); + oop obj = cast_to_oop(r->bottom()); + size_t size = obj->size(); + Thread* thread = Thread::current(); + heap->report_promotion_failure(thread, size); + heap->handle_promotion_failure(); + } } // else, region is free, or OLD, or not in collection set, or humongous_continuation, // or is young humongous_start that is too young to be promoted diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index 53e8465d89828..1fd8cfcc45f68 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -776,9 +776,11 @@ class ShenandoahHeap : public CollectedHeap { inline oop try_evacuate_object(oop src, Thread* thread, ShenandoahHeapRegion* from_region, ShenandoahRegionAffiliation target_gen); void handle_old_evacuation(HeapWord* obj, size_t words, bool promotion); void handle_old_evacuation_failure(); - void handle_promotion_failure(); public: + void handle_promotion_failure(); + void report_promotion_failure(Thread* thread, size_t size); + static address in_cset_fast_test_addr(); ShenandoahCollectionSet* collection_set() const { return _collection_set; } @@ -854,7 +856,6 @@ class ShenandoahHeap : public CollectedHeap { void try_inject_alloc_failure(); bool should_inject_alloc_failure(); - void report_promotion_failure(Thread* thread, size_t size); }; #endif // SHARE_GC_SHENANDOAH_SHENANDOAHHEAP_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index c8934769d57dc..4521de649951e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -974,16 +974,21 @@ void ShenandoahHeapRegion::set_affiliation(ShenandoahRegionAffiliation new_affil heap->old_generation()->decrement_affiliated_region_count(); } + size_t regions; switch (new_affiliation) { case FREE: assert(!has_live(), "Free region should not have live data"); break; case YOUNG_GENERATION: reset_age(); - heap->young_generation()->increment_affiliated_region_count(); + regions = heap->young_generation()->increment_affiliated_region_count(); + assert(regions * ShenandoahHeapRegion::region_size_bytes() <= heap->young_generation()->adjusted_capacity(), + "Number of young regions cannot exceed adjusted capacity"); break; case OLD_GENERATION: - heap->old_generation()->increment_affiliated_region_count(); + regions = heap->old_generation()->increment_affiliated_region_count(); + assert(regions * ShenandoahHeapRegion::region_size_bytes() <= heap->old_generation()->adjusted_capacity(), + "Number of old regions cannot exceed adjusted capacity"); break; default: ShouldNotReachHere(); @@ -992,6 +997,7 @@ void ShenandoahHeapRegion::set_affiliation(ShenandoahRegionAffiliation new_affil heap->set_affiliation(this, new_affiliation); } +// Returns number of regions promoted, or zero if we choose not to promote. size_t ShenandoahHeapRegion::promote_humongous() { ShenandoahHeap* heap = ShenandoahHeap::heap(); ShenandoahMarkingContext* marking_context = heap->marking_context(); @@ -1006,31 +1012,59 @@ size_t ShenandoahHeapRegion::promote_humongous() { oop obj = cast_to_oop(bottom()); assert(marking_context->is_marked(obj), "promoted humongous object should be alive"); + // TODO: Consider not promoting humongous objects that represent primitive arrays. Leaving a primitive array + // (obj->is_typeArray()) in young-gen is harmless because these objects are never relocated and they are not + // scanned. Leaving primitive arrays in young-gen memory allows their memory to be reclaimed more quickly when + // it becomes garbage. Better to not make this change until sizes of young-gen and old-gen are completely + // adaptive, as leaving primitive arrays in young-gen might be perceived as an "astonishing result" by someone + // has carefully analyzed the required sizes of an application's young-gen and old-gen. + size_t spanned_regions = ShenandoahHeapRegion::required_regions(obj->size() * HeapWordSize); size_t index_limit = index() + spanned_regions; - log_debug(gc)("promoting humongous region " SIZE_FORMAT ", spanning " SIZE_FORMAT, index(), spanned_regions); + { + // We need to grab the heap lock in order to avoid a race when changing the affiliations of spanned_regions from + // young to old. + ShenandoahHeapLocker locker(heap->lock()); + size_t available_old_regions = old_generation->adjusted_unaffiliated_regions(); + if (spanned_regions <= available_old_regions) { + log_debug(gc)("promoting humongous region " SIZE_FORMAT ", spanning " SIZE_FORMAT, index(), spanned_regions); + + // For this region and each humongous continuation region spanned by this humongous object, change + // affiliation to OLD_GENERATION and adjust the generation-use tallies. The remnant of memory + // in the last humongous region that is not spanned by obj is currently not used. + for (size_t i = index(); i < index_limit; i++) { + ShenandoahHeapRegion* r = heap->get_region(i); + log_debug(gc)("promoting humongous region " SIZE_FORMAT ", from " PTR_FORMAT " to " PTR_FORMAT, + r->index(), p2i(r->bottom()), p2i(r->top())); + // We mark the entire humongous object's range as dirty after loop terminates, so no need to dirty the range here + r->set_affiliation(OLD_GENERATION); + old_generation->increase_used(r->used()); + young_generation->decrease_used(r->used()); + } + // Then fall through to finish the promotion after releasing the heap lock. + } else { + // There are not enough available old regions to promote this humongous region at this time, so defer promotion. + // TODO: Consider allowing the promotion now, with the expectation that we can resize and/or collect OLD + // momentarily to address the transient violation of budgets. Some problems that need to be addressed in order + // to allow transient violation of capacity budgets are: + // 1. Various size_t subtractions assume usage is less than capacity, and thus assume there will be no + // arithmetic underflow when we subtract usage from capacity. The results of such size_t subtractions + // would need to be guarded and special handling provided. + // 2. ShenandoahVerifier enforces that usage is less than capacity. If we are going to relax this constraint, + // we need to think about what conditions allow the constraint to be violated and document and implement the + // changes. + return 0; + } + } // Since this region may have served previously as OLD, it may hold obsolete object range info. heap->card_scan()->reset_object_range(bottom(), bottom() + spanned_regions * ShenandoahHeapRegion::region_size_words()); // Since the humongous region holds only one object, no lock is necessary for this register_object() invocation. heap->card_scan()->register_object_wo_lock(bottom()); - // For this region and each humongous continuation region spanned by this humongous object, change - // affiliation to OLD_GENERATION and adjust the generation-use tallies. The remnant of memory - // in the last humongous region that is not spanned by obj is currently not used. - for (size_t i = index(); i < index_limit; i++) { - ShenandoahHeapRegion* r = heap->get_region(i); - log_debug(gc)("promoting humongous region " SIZE_FORMAT ", from " PTR_FORMAT " to " PTR_FORMAT, - r->index(), p2i(r->bottom()), p2i(r->top())); - // We mark the entire humongous object's range as dirty after loop terminates, so no need to dirty the range here - r->set_affiliation(OLD_GENERATION); - old_generation->increase_used(r->used()); - young_generation->decrease_used(r->used()); - } if (obj->is_typeArray()) { - // Primitive arrays don't need to be scanned. See above TODO question about requiring - // region promotion at safepoint. + // Primitive arrays don't need to be scanned. log_debug(gc)("Clean cards for promoted humongous object (Region " SIZE_FORMAT ") from " PTR_FORMAT " to " PTR_FORMAT, index(), p2i(bottom()), p2i(bottom() + obj->size())); heap->card_scan()->mark_range_as_clean(bottom(), obj->size()); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp index 3590ac0a83070..f7288601df4bf 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp @@ -340,23 +340,30 @@ class ShenandoahVerifyOopClosure : public BasicOopIterateClosure { virtual void do_oop(narrowOop* p) { do_oop_work(p); } }; +// This closure computes the amounts of used, committed, and garbage memory and the number of regions contained within +// a subset (e.g. the young generation or old generation) of the total heap. class ShenandoahCalculateRegionStatsClosure : public ShenandoahHeapRegionClosure { private: - size_t _used, _committed, _garbage; + size_t _used, _committed, _garbage, _regions; public: - ShenandoahCalculateRegionStatsClosure() : _used(0), _committed(0), _garbage(0) {}; + ShenandoahCalculateRegionStatsClosure() : _used(0), _committed(0), _garbage(0), _regions(0) {}; void heap_region_do(ShenandoahHeapRegion* r) { _used += r->used(); - log_debug(gc)("ShenandoahCalculatRegionStatsClosure added " SIZE_FORMAT " for %s Region " SIZE_FORMAT ", yielding: " SIZE_FORMAT, + log_debug(gc)("ShenandoahCalculateRegionStatsClosure added " SIZE_FORMAT " for %s Region " SIZE_FORMAT ", yielding: " SIZE_FORMAT, r->used(), r->is_humongous()? "humongous": "regular", r->index(), _used); _garbage += r->garbage(); _committed += r->is_committed() ? ShenandoahHeapRegion::region_size_bytes() : 0; + _regions++; } size_t used() { return _used; } size_t committed() { return _committed; } size_t garbage() { return _garbage; } + size_t regions() { return _regions; } + + // span is the total memory affiliated with these stats (some of which is in use and other is available) + size_t span() { return _regions * ShenandoahHeapRegion::region_size_bytes(); } }; class ShenandoahGenerationStatsClosure : public ShenandoahHeapRegionClosure { @@ -395,6 +402,17 @@ class ShenandoahGenerationStatsClosure : public ShenandoahHeapRegionClosure { label, generation->name(), byte_size_in_proper_unit(generation_used), proper_unit_for_byte_size(generation_used), byte_size_in_proper_unit(stats.used()), proper_unit_for_byte_size(stats.used())); + + guarantee(stats.regions() == generation->used_regions(), + "%s: generation (%s) used regions (" SIZE_FORMAT ") must equal regions that are in use (" SIZE_FORMAT ")", + label, generation->name(), generation->used_regions(), stats.regions()); + + size_t capacity = generation->adjusted_capacity(); + guarantee(stats.span() <= capacity, + "%s: generation (%s) size spanned by regions (" SIZE_FORMAT ") must not exceed current capacity (" SIZE_FORMAT "%s)", + label, generation->name(), stats.regions(), + byte_size_in_proper_unit(capacity), proper_unit_for_byte_size(capacity)); + } }; From ee49a4888452196877911f10b9b40fb08b2ae293 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Thu, 8 Dec 2022 21:45:18 +0000 Subject: [PATCH 159/254] Generation resizing Reviewed-by: rkennke, kdnilsen --- .../shenandoahAdaptiveHeuristics.cpp | 2 +- .../heuristics/shenandoahHeuristics.cpp | 6 +- .../heuristics/shenandoahHeuristics.hpp | 4 +- .../mode/shenandoahGenerationalMode.cpp | 1 + .../gc/shenandoah/shenandoahArguments.cpp | 7 + .../share/gc/shenandoah/shenandoahAsserts.hpp | 4 + .../shenandoah/shenandoahCollectorPolicy.hpp | 4 +- .../gc/shenandoah/shenandoahConcurrentGC.cpp | 1 + .../gc/shenandoah/shenandoahControlThread.cpp | 17 +- .../gc/shenandoah/shenandoahControlThread.hpp | 2 + .../gc/shenandoah/shenandoahDegeneratedGC.cpp | 2 + .../share/gc/shenandoah/shenandoahFullGC.cpp | 2 + .../gc/shenandoah/shenandoahGeneration.cpp | 24 +++ .../gc/shenandoah/shenandoahGeneration.hpp | 8 + .../share/gc/shenandoah/shenandoahHeap.cpp | 37 ++++ .../share/gc/shenandoah/shenandoahHeap.hpp | 6 + .../gc/shenandoah/shenandoahMmuTracker.cpp | 190 ++++++++++++++++++ .../gc/shenandoah/shenandoahMmuTracker.hpp | 99 +++++++++ .../share/gc/shenandoah/shenandoahUtils.cpp | 18 +- .../shenandoah/shenandoahYoungGeneration.cpp | 10 + .../shenandoah/shenandoahYoungGeneration.hpp | 4 +- .../gc/shenandoah/shenandoah_globals.hpp | 12 ++ 22 files changed, 430 insertions(+), 30 deletions(-) create mode 100644 src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp create mode 100644 src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.hpp diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp index 4225452492adc..dbcc7eabd8de9 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp @@ -419,7 +419,7 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { // ShenandoahControlIntervalMax. The current control interval (or the max control interval) should also be added into // the calculation of avg_cycle_time below. - double avg_cycle_time = _gc_time_history->davg() + (_margin_of_error_sd * _gc_time_history->dsd()); + double avg_cycle_time = _gc_cycle_time_history->davg() + (_margin_of_error_sd * _gc_cycle_time_history->dsd()); size_t last_live_memory = get_last_live_memory(); size_t penultimate_live_memory = get_penultimate_live_memory(); diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp index 5044f0617f55c..9b31e4a01fbcd 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp @@ -57,7 +57,7 @@ ShenandoahHeuristics::ShenandoahHeuristics(ShenandoahGeneration* generation) : _last_cycle_end(0), _gc_times_learned(0), _gc_time_penalties(0), - _gc_time_history(new TruncatedSeq(10, ShenandoahAdaptiveDecayFactor)), + _gc_cycle_time_history(new TruncatedSeq(10, ShenandoahAdaptiveDecayFactor)), _live_memory_last_cycle(0), _live_memory_penultimate_cycle(0), _metaspace_oom() @@ -302,7 +302,7 @@ void ShenandoahHeuristics::record_success_concurrent(bool abbreviated) { _successful_cycles_in_a_row++; if (!(abbreviated && ShenandoahAdaptiveIgnoreShortCycles)) { - _gc_time_history->add(time_since_last_gc()); + _gc_cycle_time_history->add(elapsed_cycle_time()); _gc_times_learned++; } @@ -361,7 +361,7 @@ void ShenandoahHeuristics::initialize() { // Nothing to do by default. } -double ShenandoahHeuristics::time_since_last_gc() const { +double ShenandoahHeuristics::elapsed_cycle_time() const { return os::elapsedTime() - _cycle_start; } diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp index 27c62b73ce366..dcafb3424777a 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp @@ -97,7 +97,7 @@ class ShenandoahHeuristics : public CHeapObj { size_t _gc_times_learned; intx _gc_time_penalties; - TruncatedSeq* _gc_time_history; + TruncatedSeq* _gc_cycle_time_history; size_t _live_memory_last_cycle; size_t _live_memory_penultimate_cycle; @@ -168,7 +168,7 @@ class ShenandoahHeuristics : public CHeapObj { virtual bool is_experimental() = 0; virtual void initialize(); - double time_since_last_gc() const; + double elapsed_cycle_time() const; void save_last_live_memory(size_t live_memory); size_t get_last_live_memory(); diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp index 640eb9b869bd3..1600b187e81ac 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp @@ -36,6 +36,7 @@ void ShenandoahGenerationalMode::initialize_flags() const { FLAG_SET_DEFAULT(VerifyBeforeExit, false); } + SHENANDOAH_ERGO_OVERRIDE_DEFAULT(GCTimeRatio, 70); SHENANDOAH_ERGO_OVERRIDE_DEFAULT(NewRatio, 1); SHENANDOAH_ERGO_OVERRIDE_DEFAULT(ShenandoahUnloadClassesFrequency, 0); SHENANDOAH_ERGO_ENABLE_FLAG(ExplicitGCInvokesConcurrent); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp b/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp index fbe3fcc486ac4..0d05cb936dea9 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp @@ -69,6 +69,13 @@ void ShenandoahArguments::initialize() { FLAG_SET_DEFAULT(UseNUMA, true); } + // We use this as the time period for tracking minimum mutator utilization (MMU). + // In generational mode, the MMU is used as a signal to adjust the size of the + // young generation. + if (FLAG_IS_DEFAULT(GCPauseIntervalMillis)) { + FLAG_SET_DEFAULT(GCPauseIntervalMillis, 5000); + } + // Set up default number of concurrent threads. We want to have cycles complete fast // enough, but we also do not want to steal too much CPU from the concurrently running // application. Using 1/4 of available threads for concurrent GC seems a good diff --git a/src/hotspot/share/gc/shenandoah/shenandoahAsserts.hpp b/src/hotspot/share/gc/shenandoah/shenandoahAsserts.hpp index f2b88fea00873..86a3cf891615d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahAsserts.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahAsserts.hpp @@ -167,6 +167,9 @@ class ShenandoahAsserts { #define shenandoah_assert_heaplocked_or_fullgc_safepoint() \ ShenandoahAsserts::assert_heaplocked_or_fullgc_safepoint(__FILE__, __LINE__) + +#define shenandoah_assert_generational() \ + assert(ShenandoahHeap::heap()->mode()->is_generational(), "Must be in generational mode here.") #else #define shenandoah_assert_in_heap(interior_loc, obj) #define shenandoah_assert_in_heap_or_null(interior_loc, obj) @@ -218,6 +221,7 @@ class ShenandoahAsserts { #define shenandoah_assert_not_heaplocked() #define shenandoah_assert_heaplocked_or_safepoint() #define shenandoah_assert_heaplocked_or_fullgc_safepoint() +#define shenandoah_assert_generational() #endif diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp index 9336b0295baef..1f8265a0e4acf 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp @@ -52,14 +52,12 @@ class ShenandoahCollectorPolicy : public CHeapObj { size_t _explicit_full; size_t _implicit_concurrent; size_t _implicit_full; + size_t _cycle_counter; size_t _degen_points[ShenandoahGC::_DEGENERATED_LIMIT]; ShenandoahSharedFlag _in_shutdown; - ShenandoahTracer* _tracer; - size_t _cycle_counter; - public: ShenandoahCollectorPolicy(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index 6ce1e0ccf3ca2..1e04ad0e15b87 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -1202,6 +1202,7 @@ void ShenandoahConcurrentGC::op_final_updaterefs() { } heap->rebuild_free_set(true /*concurrent*/); + heap->adjust_generation_sizes(); } void ShenandoahConcurrentGC::op_final_roots() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index 498909bab8459..339205429dffd 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -448,7 +448,7 @@ void ShenandoahControlThread::process_phase_timings(const ShenandoahHeap* heap) // void ShenandoahControlThread::service_concurrent_normal_cycle( const ShenandoahHeap* heap, const GenerationMode generation, GCCause::Cause cause) { - + GCIdMark gc_id_mark; switch (generation) { case YOUNG: { // Run a young cycle. This might or might not, have interrupted an ongoing @@ -458,24 +458,22 @@ void ShenandoahControlThread::service_concurrent_normal_cycle( // that are in the cset. log_info(gc, ergo)("Start GC cycle (YOUNG)"); service_concurrent_cycle(heap->young_generation(), cause, false); - heap->young_generation()->log_status(); break; } case GLOBAL: { log_info(gc, ergo)("Start GC cycle (GLOBAL)"); service_concurrent_cycle(heap->global_generation(), cause, false); - heap->global_generation()->log_status(); break; } case OLD: { log_info(gc, ergo)("Start GC cycle (OLD)"); service_concurrent_old_cycle(heap, cause); - heap->old_generation()->log_status(); break; } default: ShouldNotReachHere(); } + log_heap_status(heap); } void ShenandoahControlThread::service_concurrent_old_cycle(const ShenandoahHeap* heap, GCCause::Cause &cause) { @@ -483,7 +481,6 @@ void ShenandoahControlThread::service_concurrent_old_cycle(const ShenandoahHeap* ShenandoahOldGeneration* old_generation = heap->old_generation(); ShenandoahYoungGeneration* young_generation = heap->young_generation(); - GCIdMark gc_id_mark; TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); switch (old_generation->state()) { @@ -582,6 +579,15 @@ bool ShenandoahControlThread::resume_concurrent_old_cycle(ShenandoahGeneration* return true; } +void ShenandoahControlThread::log_heap_status(const ShenandoahHeap* heap) { + if (heap->mode()->is_generational()) { + heap->young_generation()->log_status(); + heap->old_generation()->log_status(); + } else { + heap->global_generation()->log_status(); + } +} + bool ShenandoahControlThread::check_soft_max_changed() const { ShenandoahHeap* heap = ShenandoahHeap::heap(); size_t new_soft_max = Atomic::load(&SoftMaxHeapSize); @@ -640,7 +646,6 @@ void ShenandoahControlThread::service_concurrent_cycle(ShenandoahGeneration* gen if (check_cancellation_or_degen(ShenandoahGC::_degenerated_outside_cycle)) return; ShenandoahHeap* heap = ShenandoahHeap::heap(); - GCIdMark gc_id_mark; ShenandoahGCSession session(cause, generation); TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp index 4bc7d89f755a2..3af9112ffe86f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp @@ -189,6 +189,8 @@ class ShenandoahControlThread: public ConcurrentGCThread { void service_concurrent_cycle(const ShenandoahHeap* heap, ShenandoahGeneration* generation, GCCause::Cause &cause, bool do_old_gc_bootstrap); + + void log_heap_status(const ShenandoahHeap* heap); }; #endif // SHARE_GC_SHENANDOAH_SHENANDOAHCONTROLTHREAD_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp index c949f4cb3d297..91368b978985a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp @@ -296,6 +296,8 @@ void ShenandoahDegenGC::op_degenerated() { heap->set_old_evac_reserve(0); heap->reset_old_evac_expended(); heap->set_promoted_reserve(0); + + heap->adjust_generation_sizes(); } if (ShenandoahVerify) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index 444614bbbe17f..3522415b97780 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -336,6 +336,8 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) { // Resize metaspace MetaspaceGC::compute_new_size(); + heap->adjust_generation_sizes(); + // Free worker slices for (uint i = 0; i < heap->max_workers(); i++) { delete worker_slices[i]; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index 9643485b8cdb9..23e3d21fb2c20 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -988,6 +988,19 @@ size_t ShenandoahGeneration::adjusted_unaffiliated_regions() const { return (adjusted_capacity() - used_regions_size()) / ShenandoahHeapRegion::region_size_bytes(); } + +void ShenandoahGeneration::increase_capacity(size_t increment) { + shenandoah_assert_heaplocked_or_safepoint(); + _max_capacity += increment; + _soft_max_capacity += increment; +} + +void ShenandoahGeneration::decrease_capacity(size_t decrement) { + shenandoah_assert_heaplocked_or_safepoint(); + _max_capacity -= decrement; + _soft_max_capacity -= decrement; +} + void ShenandoahGeneration::record_success_concurrent(bool abbreviated) { heuristics()->record_success_concurrent(abbreviated); ShenandoahHeap::heap()->shenandoah_policy()->record_success_concurrent(); @@ -997,3 +1010,14 @@ void ShenandoahGeneration::record_success_degenerated() { heuristics()->record_success_degenerated(); ShenandoahHeap::heap()->shenandoah_policy()->record_success_degenerated(); } + +void ShenandoahGeneration::add_collection_time(double time_seconds) { + _collection_thread_time_s += time_seconds; +} + +double ShenandoahGeneration::reset_collection_time() { + double t = _collection_thread_time_s; + _collection_thread_time_s = 0.0; + return t; +} + diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index d3941a37c0af4..d915149bb54ff 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -46,6 +46,8 @@ class ShenandoahGeneration : public CHeapObj { ShenandoahReferenceProcessor* const _ref_processor; + double _collection_thread_time_s; + protected: // Usage size_t _affiliated_region_count; @@ -109,6 +111,9 @@ class ShenandoahGeneration : public CHeapObj { void reset_bytes_allocated_since_gc_start(); void increase_allocated(size_t bytes); + void increase_capacity(size_t increment); + void decrease_capacity(size_t decrement); + void set_soft_max_capacity(size_t soft_max_capacity) { _soft_max_capacity = soft_max_capacity; } @@ -181,6 +186,9 @@ class ShenandoahGeneration : public CHeapObj { virtual void record_success_concurrent(bool abbreviated); virtual void record_success_degenerated(); + + virtual void add_collection_time(double time_seconds); + double reset_collection_time(); }; #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHGENERATION_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index caa993b9600aa..15fac82f2d10a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -625,6 +625,8 @@ class ShenandoahInitWorkerGCLABClosure : public ThreadClosure { void ShenandoahHeap::post_initialize() { CollectedHeap::post_initialize(); + _mmu_tracker.initialize(); + MutexLocker ml(Threads_lock); ShenandoahInitWorkerGCLABClosure init_gclabs; @@ -1093,6 +1095,13 @@ void ShenandoahHeap::coalesce_and_fill_old_regions() { parallel_heap_region_iterate(&coalesce); } +bool ShenandoahHeap::adjust_generation_sizes() { + if (mode()->is_generational()) { + return _mmu_tracker.adjust_generation_sizes(); + } + return false; +} + HeapWord* ShenandoahHeap::allocate_new_tlab(size_t min_size, size_t requested_size, size_t* actual_size) { @@ -1741,6 +1750,7 @@ void ShenandoahHeap::prepare_for_verify() { void ShenandoahHeap::gc_threads_do(ThreadClosure* tcl) const { tcl->do_thread(_control_thread); + tcl->do_thread(_regulator_thread); workers()->threads_do(tcl); if (_safepoint_workers != NULL) { _safepoint_workers->threads_do(tcl); @@ -1772,6 +1782,33 @@ void ShenandoahHeap::print_tracing_info() const { } } +void ShenandoahHeap::on_cycle_start(GCCause::Cause cause, ShenandoahGeneration* generation) { + set_gc_cause(cause); + set_gc_generation(generation); + + shenandoah_policy()->record_cycle_start(); + generation->heuristics()->record_cycle_start(); + + // When a cycle starts, attribute any thread activity when the collector + // is idle to the global generation. + _mmu_tracker.record(global_generation()); +} + +void ShenandoahHeap::on_cycle_end(ShenandoahGeneration* generation) { + generation->heuristics()->record_cycle_end(); + + if (mode()->is_generational() && + ((generation->generation_mode() == GLOBAL) || upgraded_to_full())) { + // If we just completed a GLOBAL GC, claim credit for completion of young-gen and old-gen GC as well + young_generation()->heuristics()->record_cycle_end(); + old_generation()->heuristics()->record_cycle_end(); + } + set_gc_cause(GCCause::_no_gc); + + // When a cycle ends, the thread activity is attributed to the respective generation + _mmu_tracker.record(generation); +} + void ShenandoahHeap::verify(VerifyOption vo) { if (ShenandoahSafepoint::is_at_shenandoah_safepoint()) { if (ShenandoahVerify) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index 1fd8cfcc45f68..239469ea41dd4 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -34,6 +34,7 @@ #include "gc/shenandoah/shenandoahLock.hpp" #include "gc/shenandoah/shenandoahEvacOOMHandler.hpp" #include "gc/shenandoah/shenandoahEvacTracker.hpp" +#include "gc/shenandoah/shenandoahMmuTracker.hpp" #include "gc/shenandoah/shenandoahPadding.hpp" #include "gc/shenandoah/shenandoahSharedVariables.hpp" #include "gc/shenandoah/shenandoahUnload.hpp" @@ -535,6 +536,7 @@ class ShenandoahHeap : public CollectedHeap { ShenandoahPhaseTimings* _phase_timings; ShenandoahEvacuationTracker* _evac_tracker; + ShenandoahMmuTracker _mmu_tracker; ShenandoahControlThread* control_thread() { return _control_thread; } ShenandoahRegulatorThread* regulator_thread() { return _regulator_thread; } @@ -553,6 +555,9 @@ class ShenandoahHeap : public CollectedHeap { ShenandoahPhaseTimings* phase_timings() const { return _phase_timings; } ShenandoahEvacuationTracker* evac_tracker() const { return _evac_tracker; } + void on_cycle_start(GCCause::Cause cause, ShenandoahGeneration* generation); + void on_cycle_end(ShenandoahGeneration* generation); + ShenandoahVerifier* verifier(); // ---------- VM subsystem bindings @@ -817,6 +822,7 @@ class ShenandoahHeap : public CollectedHeap { void cancel_old_gc(); bool is_old_gc_active(); void coalesce_and_fill_old_regions(); + bool adjust_generation_sizes(); // ---------- Helper functions // diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp new file mode 100644 index 0000000000000..05ff09ffecf4b --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp @@ -0,0 +1,190 @@ +/* + * Copyright (c) 2022, Amazon, Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ +#include "precompiled.hpp" + +#include "gc/shenandoah/shenandoahMmuTracker.hpp" +#include "gc/shenandoah/shenandoahHeap.inline.hpp" +#include "gc/shenandoah/shenandoahOldGeneration.hpp" +#include "gc/shenandoah/shenandoahYoungGeneration.hpp" +#include "runtime/os.hpp" +#include "runtime/task.hpp" +#include "logging/log.hpp" + + +class ShenandoahMmuTask : public PeriodicTask { + ShenandoahMmuTracker* _mmu_tracker; +public: + ShenandoahMmuTask(ShenandoahMmuTracker* mmu_tracker) : + PeriodicTask(GCPauseIntervalMillis), _mmu_tracker(mmu_tracker) {} + + virtual void task() override { + _mmu_tracker->report(); + } +}; + +class ThreadTimeAccumulator : public ThreadClosure { + public: + size_t total_time; + ThreadTimeAccumulator() : total_time(0) {} + virtual void do_thread(Thread* thread) override { + total_time += os::thread_cpu_time(thread); + } +}; + +double ShenandoahMmuTracker::gc_thread_time_seconds() { + ThreadTimeAccumulator cl; + ShenandoahHeap::heap()->gc_threads_do(&cl); + // Include VM thread? Compiler threads? or no - because there + // is nothing the collector can do about those threads. + return double(cl.total_time) / NANOSECS_PER_SEC; +} + +double ShenandoahMmuTracker::process_time_seconds() { + double process_real_time(0.0), process_user_time(0.0), process_system_time(0.0); + bool valid = os::getTimesSecs(&process_real_time, &process_user_time, &process_system_time); + if (valid) { + return process_user_time + process_system_time; + } + return 0.0; +} + +ShenandoahMmuTracker::ShenandoahMmuTracker() : + _initial_collector_time_s(0.0), + _initial_process_time_s(0.0), + _resize_increment(YoungGenerationSizeIncrement / 100.0), + _mmu_periodic_task(new ShenandoahMmuTask(this)), + _mmu_average(10, ShenandoahAdaptiveDecayFactor) { +} + +ShenandoahMmuTracker::~ShenandoahMmuTracker() { + _mmu_periodic_task->disenroll(); + delete _mmu_periodic_task; +} + +void ShenandoahMmuTracker::record(ShenandoahGeneration* generation) { + // This is only called by the control thread. + double collector_time_s = gc_thread_time_seconds(); + double elapsed_gc_time_s = collector_time_s - _initial_collector_time_s; + generation->add_collection_time(elapsed_gc_time_s); + _initial_collector_time_s = collector_time_s; +} + +void ShenandoahMmuTracker::report() { + // This is only called by the periodic thread. + double process_time_s = process_time_seconds(); + double elapsed_process_time_s = process_time_s - _initial_process_time_s; + _initial_process_time_s = process_time_s; + double verify_time_s = gc_thread_time_seconds(); + double verify_elapsed = verify_time_s - _initial_verify_collector_time_s; + _initial_verify_collector_time_s = verify_time_s; + double verify_mmu = ((elapsed_process_time_s - verify_elapsed) / elapsed_process_time_s) * 100; + _mmu_average.add(verify_mmu); + log_info(gc)("Average MMU = %.3f", _mmu_average.davg()); +} + +bool ShenandoahMmuTracker::adjust_generation_sizes() { + shenandoah_assert_generational(); + if (_mmu_average.davg() >= double(GCTimeRatio)) { + return false; + } + + ShenandoahHeap* heap = ShenandoahHeap::heap(); + ShenandoahOldGeneration *old = heap->old_generation(); + double old_time_s = old->reset_collection_time(); + ShenandoahYoungGeneration *young = heap->young_generation(); + double young_time_s = young->reset_collection_time(); + ShenandoahGeneration *global = heap->global_generation(); + double global_time_s = global->reset_collection_time(); + + log_info(gc)("Thread Usr+Sys YOUNG = %.3f, OLD = %.3f, GLOBAL = %.3f", young_time_s, old_time_s, global_time_s); + + if (old_time_s > young_time_s) { + return transfer_capacity(young, old); + } else { + return transfer_capacity(old, young); + } +} + +size_t percentage_of_heap(size_t bytes) { + size_t heap_capacity = ShenandoahHeap::heap()->max_capacity(); + assert(bytes < heap_capacity, "Must be less than total capacity"); + return size_t(100.0 * double(bytes) / double(heap_capacity)); +} + +bool ShenandoahMmuTracker::transfer_capacity(ShenandoahGeneration* from, ShenandoahGeneration* to) { + shenandoah_assert_heaplocked_or_safepoint(); + + size_t available_regions = from->free_unaffiliated_regions(); + if (available_regions <= 0) { + log_info(gc)("%s has no regions available for transfer to %s", from->name(), to->name()); + return false; + } + + size_t regions_to_transfer = MAX2(1UL, size_t(double(available_regions) * _resize_increment)); + size_t bytes_to_transfer = regions_to_transfer * ShenandoahHeapRegion::region_size_bytes(); + if (from->generation_mode() == YOUNG) { + size_t new_young_size = from->max_capacity() - bytes_to_transfer; + if (percentage_of_heap(new_young_size) < ShenandoahMinYoungPercentage) { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + size_t minimum_size = size_t(ShenandoahMinYoungPercentage / 100.0 * heap->max_capacity()); + if (from->max_capacity() > minimum_size) { + bytes_to_transfer = from->max_capacity() - minimum_size; + } else { + log_info(gc)("Cannot transfer from young: " SIZE_FORMAT "%s, at minimum capacity: " SIZE_FORMAT "%s", + byte_size_in_proper_unit(from->max_capacity()), proper_unit_for_byte_size(from->max_capacity()), + byte_size_in_proper_unit(minimum_size), proper_unit_for_byte_size(minimum_size)); + return false; + } + } + } else { + assert(to->generation_mode() == YOUNG, "Can only transfer between young and old."); + size_t new_young_size = to->max_capacity() + bytes_to_transfer; + if (percentage_of_heap(new_young_size) > ShenandoahMaxYoungPercentage) { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + size_t maximum_size = size_t(ShenandoahMaxYoungPercentage / 100.0 * heap->max_capacity()); + if (maximum_size > to->max_capacity()) { + bytes_to_transfer = maximum_size - to->max_capacity(); + } else { + log_info(gc)("Cannot transfer to young: " SIZE_FORMAT "%s, at maximum capacity: " SIZE_FORMAT "%s", + byte_size_in_proper_unit(to->max_capacity()), proper_unit_for_byte_size(to->max_capacity()), + byte_size_in_proper_unit(maximum_size), proper_unit_for_byte_size(maximum_size)); + return false; + } + } + } + + assert(bytes_to_transfer <= regions_to_transfer * ShenandoahHeapRegion::region_size_bytes(), "Cannot transfer more than available in free regions."); + log_info(gc)("Transfer " SIZE_FORMAT "%s from %s to %s", byte_size_in_proper_unit(bytes_to_transfer), + proper_unit_for_byte_size(bytes_to_transfer), from->name(), to->name()); + from->decrease_capacity(bytes_to_transfer); + to->increase_capacity(bytes_to_transfer); + return true; +} + +void ShenandoahMmuTracker::initialize() { + _initial_process_time_s = process_time_seconds(); + _initial_collector_time_s = gc_thread_time_seconds(); + _initial_verify_collector_time_s = _initial_collector_time_s; + _mmu_periodic_task->enroll(); +} \ No newline at end of file diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.hpp new file mode 100644 index 0000000000000..1c9f20d9d2400 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.hpp @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2022, Amazon, Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHMMUTRACKER_HPP +#define SHARE_GC_SHENANDOAH_SHENANDOAHMMUTRACKER_HPP + +#include "runtime/mutex.hpp" +#include "utilities/numberSeq.hpp" + +class ShenandoahGeneration; +class ShenandoahMmuTask; + +/** + * This class is responsible for tracking and adjusting the minimum mutator + * utilization (MMU). MMU is defined as the percentage of CPU time available + * to mutator threads over an arbitrary, fixed interval of time. MMU is measured + * by summing all of the time given to the GC threads and comparing this too + * the total CPU time for the process. There are OS APIs to support this on + * all major platforms. + * + * The time spent by GC threads is attributed to the young or old generation. + * The time given to the controller and regulator threads is attributed to the + * global generation. At the end of every collection, the average MMU is inspected. + * If it is below `GCTimeRatio`, this class will attempt to increase the capacity + * of the generation that is consuming the most CPU time. The assumption being + * that increasing memory will reduce the collection frequency and raise the + * MMU. + */ +class ShenandoahMmuTracker { + + double _initial_collector_time_s; + double _initial_process_time_s; + double _initial_verify_collector_time_s; + + double _resize_increment; + + ShenandoahMmuTask* _mmu_periodic_task; + TruncatedSeq _mmu_average; + + bool transfer_capacity(ShenandoahGeneration* from, ShenandoahGeneration* to); + + static double gc_thread_time_seconds(); + static double process_time_seconds(); + +public: + explicit ShenandoahMmuTracker(); + ~ShenandoahMmuTracker(); + + // This enrolls the periodic task after everything is initialized. + void initialize(); + + // This is called at the start and end of a GC cycle. The GC thread times + // will be accumulated in this generation. Note that the bootstrap cycle + // for an old collection should be counted against the old generation. + // When the collector is idle, it still runs a regulator and a control. + // The times for these threads are attributed to the global generation. + void record(ShenandoahGeneration* generation); + + // This is called by the periodic task timer. The interval is defined by + // GCPauseIntervalMillis and defaults to 5 seconds. This method computes + // the MMU over the elapsed interval and records it in a running average. + // This method also logs the average MMU. + void report(); + + // This is invoked at the end of a collection. This happens on a safepoint + // to avoid any races with allocators (and to avoid interfering with + // allocators by taking the heap lock). The amount of capacity to move + // from one generation to another is controlled by YoungGenerationSizeIncrement + // and defaults to 20% of the heap. The minimum and maximum sizes of the + // young generation are controlled by ShenandoahMinYoungPercentage and + // ShenandoahMaxYoungPercentage, respectively. The method returns true + // when and adjustment is made, false otherwise. + bool adjust_generation_sizes(); +}; + + + +#endif //SHARE_GC_SHENANDOAH_SHENANDOAHMMUTRACKER_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp b/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp index 4fbdd69ac9f93..f2239a78b86ad 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp @@ -47,14 +47,12 @@ ShenandoahGCSession::ShenandoahGCSession(GCCause::Cause cause, ShenandoahGenerat _tracer(_heap->tracer()) { assert(!ShenandoahGCPhase::is_current_phase_valid(), "No current GC phase"); - _heap->set_gc_cause(cause); - _heap->set_gc_generation(generation); + _heap->on_cycle_start(cause, _generation); + _timer->register_gc_start(); _tracer->report_gc_start(cause, _timer->gc_start()); _heap->trace_heap_before_gc(_tracer); - _heap->shenandoah_policy()->record_cycle_start(); - generation->heuristics()->record_cycle_start(); _trace_cycle.initialize(_heap->cycle_memory_manager(), cause, /* allMemoryPoolsAffected */ true, /* recordGCBeginTime = */ true, @@ -67,22 +65,14 @@ ShenandoahGCSession::ShenandoahGCSession(GCCause::Cause cause, ShenandoahGenerat ); } - ShenandoahGCSession::~ShenandoahGCSession() { - - _generation->heuristics()->record_cycle_end(); - if (_heap->mode()->is_generational() && - ((_generation->generation_mode() == GLOBAL) || _heap->upgraded_to_full())) { - // If we just completed a GLOBAL GC, claim credit for completion of young-gen and old-gen GC as well - _heap->young_generation()->heuristics()->record_cycle_end(); - _heap->old_generation()->heuristics()->record_cycle_end(); - } + _heap->on_cycle_end(_generation); _timer->register_gc_end(); _heap->trace_heap_after_gc(_tracer); _tracer->report_gc_reference_stats(_generation->ref_processor()->reference_process_stats()); _tracer->report_gc_end(_timer->gc_end(), _timer->time_partitions()); assert(!ShenandoahGCPhase::is_current_phase_valid(), "No current GC phase"); - _heap->set_gc_cause(GCCause::_no_gc); + } ShenandoahGCPauseMark::ShenandoahGCPauseMark(uint gc_id, SvcGCMarker::reason_type type) : diff --git a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp index 2c4f0003f733c..f7f70bfdf1b6b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp @@ -26,6 +26,7 @@ #include "gc/shenandoah/shenandoahFreeSet.hpp" #include "gc/shenandoah/shenandoahHeap.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" +#include "gc/shenandoah/shenandoahOldGeneration.hpp" #include "gc/shenandoah/shenandoahUtils.hpp" #include "gc/shenandoah/shenandoahVerifier.hpp" #include "gc/shenandoah/shenandoahYoungGeneration.hpp" @@ -89,3 +90,12 @@ ShenandoahHeuristics* ShenandoahYoungGeneration::initialize_heuristics(Shenandoa confirm_heuristics_mode(); return _heuristics; } + +void ShenandoahYoungGeneration::add_collection_time(double time_seconds) { + if (_old_gen_task_queues != NULL) { + // This is a bootstrap cycle, so attribute time to old gc + ShenandoahHeap::heap()->old_generation()->add_collection_time(time_seconds); + } else { + ShenandoahGeneration::add_collection_time(time_seconds); + } +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp index 6fb2cb262d183..bf517647aa3ca 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp @@ -56,7 +56,9 @@ class ShenandoahYoungGeneration : public ShenandoahGeneration { virtual ShenandoahHeuristics* initialize_heuristics(ShenandoahMode* gc_mode) override; -protected: + virtual void add_collection_time(double time_seconds) override; + + protected: bool is_concurrent_mark_in_progress() override; }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp index 0befadcc745c0..e8e9fd8172cd2 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp @@ -376,6 +376,18 @@ "events.") \ range(0,100) \ \ + product(uintx, ShenandoahMinYoungPercentage, 20, \ + "The minimum percentage of the heap to use for the young " \ + "generation. Heuristics will not adjust the young generation " \ + "to be less than this.") \ + range(0, 100) \ + \ + product(uintx, ShenandoahMaxYoungPercentage, 80, \ + "The maximum percentage of the heap to use for the young " \ + "generation. Heuristics will not adjust the young generation " \ + "to be more than this.") \ + range(0, 100) \ + \ product(bool, ShenandoahPacing, true, EXPERIMENTAL, \ "Pace application allocations to give GC chance to start " \ "and complete before allocation failure is reached.") \ From 7a3ebbcdae1659ba9bb9be04e6419aa3b34cc8c9 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Wed, 14 Dec 2022 16:50:30 +0000 Subject: [PATCH 160/254] Generation sizing fixes Reviewed-by: ysr, kdnilsen --- src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp | 1 + src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index 339205429dffd..223f2afd28a1b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -292,6 +292,7 @@ void ShenandoahControlThread::run_service() { } case servicing_old: { assert(generation == OLD, "Expected old generation here"); + GCIdMark gc_id_mark; service_concurrent_old_cycle(heap, cause); break; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp index 05ff09ffecf4b..4043145e07ff6 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp @@ -141,7 +141,7 @@ bool ShenandoahMmuTracker::transfer_capacity(ShenandoahGeneration* from, Shenand return false; } - size_t regions_to_transfer = MAX2(1UL, size_t(double(available_regions) * _resize_increment)); + size_t regions_to_transfer = MAX2(1u, uint(double(available_regions) * _resize_increment)); size_t bytes_to_transfer = regions_to_transfer * ShenandoahHeapRegion::region_size_bytes(); if (from->generation_mode() == YOUNG) { size_t new_young_size = from->max_capacity() - bytes_to_transfer; From 35b26d605a61cba864458e4493bf80fe7fda31ad Mon Sep 17 00:00:00 2001 From: William Kemper Date: Wed, 14 Dec 2022 16:51:46 +0000 Subject: [PATCH 161/254] Allow adjusted capacity and used regions size to be equal Reviewed-by: ysr, kdnilsen --- src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index 23e3d21fb2c20..7eab9636f7544 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -984,7 +984,7 @@ size_t ShenandoahGeneration::adjusted_capacity() const { } size_t ShenandoahGeneration::adjusted_unaffiliated_regions() const { - assert(adjusted_capacity() > used_regions_size(), "adjusted_unaffiliated_regions() cannot return negative"); + assert(adjusted_capacity() >= used_regions_size(), "adjusted_unaffiliated_regions() cannot return negative"); return (adjusted_capacity() - used_regions_size()) / ShenandoahHeapRegion::region_size_bytes(); } From bbd4ef345122aeb2277c5f24269bda11846ec6ef Mon Sep 17 00:00:00 2001 From: "Y. Srinivas Ramakrishna" Date: Mon, 19 Dec 2022 20:55:59 +0000 Subject: [PATCH 162/254] 8298597: HdrSeq: support for a merge() method Reviewed-by: kdnilsen --- .../gc/shenandoah/shenandoahNumberSeq.cpp | 36 ++++++ .../gc/shenandoah/shenandoahNumberSeq.hpp | 3 + src/hotspot/share/utilities/numberSeq.cpp | 44 +++++++ src/hotspot/share/utilities/numberSeq.hpp | 9 ++ .../shenandoah/test_shenandoahNumberSeq.cpp | 113 +++++++++++++----- 5 files changed, 178 insertions(+), 27 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.cpp b/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.cpp index c0fc3aca5a7c6..a460f03edace1 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.cpp @@ -120,6 +120,42 @@ double HdrSeq::percentile(double level) const { return maximum(); } +// Merge this HdrSeq into hdr2: clear optional and on-by-default +// Note: this method isn't intrinsically MT-safe; callers must take care +// of any mutual exclusion as necessary. +void HdrSeq::merge(HdrSeq& hdr2, bool clear_this) { + for (int mag = 0; mag < MagBuckets; mag++) { + if (_hdr[mag] != NULL) { + int* that_bucket = hdr2._hdr[mag]; + if (that_bucket == NULL) { + if (clear_this) { + // the target doesn't have any values, swap in ours. + // Could this cause native memory fragmentation? + hdr2._hdr[mag] = _hdr[mag]; + _hdr[mag] = NULL; + } else { + // We can't clear this, so we create the entries & add in below + that_bucket = NEW_C_HEAP_ARRAY(int, ValBuckets, mtInternal); + for (int val = 0; val < ValBuckets; val++) { + that_bucket[val] = _hdr[mag][val]; + } + hdr2._hdr[mag] = that_bucket; + } + } else { + // Add in our values into target + for (int val = 0; val < ValBuckets; val++) { + that_bucket[val] += _hdr[mag][val]; + if (clear_this) { + _hdr[mag][val] = 0; + } + } + } + } + } + // Merge up the class hierarchy + NumberSeq::merge(hdr2, clear_this); +} + BinaryMagnitudeSeq::BinaryMagnitudeSeq() { _mags = NEW_C_HEAP_ARRAY(size_t, BitsPerSize_t, mtInternal); clear(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.hpp b/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.hpp index 42f91f6a9b8b3..6dd4cd6a93fb1 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.hpp @@ -50,6 +50,9 @@ class HdrSeq: public NumberSeq { virtual void add(double val); double percentile(double level) const; + + // Merge this HdrSeq into hdr2, optionally clearing this HdrSeq + void merge(HdrSeq& hdr2, bool clear_this = true); }; // Binary magnitude sequence stores the power-of-two histogram. diff --git a/src/hotspot/share/utilities/numberSeq.cpp b/src/hotspot/share/utilities/numberSeq.cpp index 0098327c75159..0fd9b8612a992 100644 --- a/src/hotspot/share/utilities/numberSeq.cpp +++ b/src/hotspot/share/utilities/numberSeq.cpp @@ -110,6 +110,34 @@ double AbsSeq::dsd() const { return sqrt(var); } + +void AbsSeq::merge(AbsSeq& abs2, bool clear_this) { + + if (num() == 0) return; // nothing to do + + abs2._num += _num; + abs2._sum += _sum; + abs2._sum_of_squares += _sum_of_squares; + + // Decaying stats need a bit more thought + assert(abs2._alpha == _alpha, "Caution: merge incompatible?"); + + // Until JDK-... is fixed, we taint the decaying statistics + if (abs2._davg != NAN) { + abs2._davg = NAN; + abs2._dvariance = NAN; + } + + if (clear_this) { + _num = 0; + _sum = 0; + _sum_of_squares = 0; + _davg = 0; + _dvariance = 0; + } +} + + NumberSeq::NumberSeq(double alpha) : AbsSeq(alpha), _last(0.0), _maximum(0.0) { } @@ -137,6 +165,22 @@ void NumberSeq::add(double val) { ++_num; } +void NumberSeq::merge(NumberSeq& nseq2, bool clear_this) { + + if (num() == 0) return; // nothing to do + + nseq2._last = _last; // this is newer than that + nseq2._maximum = MAX2(_maximum, nseq2._maximum); + + AbsSeq::merge(nseq2, clear_this); + + if (clear_this) { + _last = 0; + _maximum = 0; + assert(num() == 0, "Not cleared"); + } +} + TruncatedSeq::TruncatedSeq(int length, double alpha): AbsSeq(alpha), _length(length), _next(0) { diff --git a/src/hotspot/share/utilities/numberSeq.hpp b/src/hotspot/share/utilities/numberSeq.hpp index b57fdf45576ab..b0ced71e79f05 100644 --- a/src/hotspot/share/utilities/numberSeq.hpp +++ b/src/hotspot/share/utilities/numberSeq.hpp @@ -83,6 +83,9 @@ class AbsSeq: public CHeapObj { // Debugging/Printing virtual void dump(); virtual void dump_on(outputStream* s); + + // Merge this AbsSeq into seq2, optionally clearing this AbsSeq + void merge(AbsSeq& seq2, bool clear_this = true); }; class NumberSeq: public AbsSeq { @@ -102,6 +105,9 @@ class NumberSeq: public AbsSeq { // Debugging/Printing virtual void dump_on(outputStream* s); + + // Merge this NumberSeq into seq2, optionally clearing this NumberSeq + void merge(NumberSeq& seq2, bool clear_this = true); }; class TruncatedSeq: public AbsSeq { @@ -129,6 +135,9 @@ class TruncatedSeq: public AbsSeq { // Debugging/Printing virtual void dump_on(outputStream* s); + + // Merge this AbsSeq into seq2, optionally clearing this AbsSeq + void merge(AbsSeq& seq2, bool clear_this = true); }; #endif // SHARE_UTILITIES_NUMBERSEQ_HPP diff --git a/test/hotspot/gtest/gc/shenandoah/test_shenandoahNumberSeq.cpp b/test/hotspot/gtest/gc/shenandoah/test_shenandoahNumberSeq.cpp index eddc11daca193..bda0a5cc78143 100644 --- a/test/hotspot/gtest/gc/shenandoah/test_shenandoahNumberSeq.cpp +++ b/test/hotspot/gtest/gc/shenandoah/test_shenandoahNumberSeq.cpp @@ -1,6 +1,5 @@ /* - * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. - * Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2022, Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -30,45 +29,105 @@ class ShenandoahNumberSeqTest: public ::testing::Test { protected: - HdrSeq seq; + const double err = 0.5; + + HdrSeq seq1; + HdrSeq seq2; + HdrSeq seq3; + + void print() { + if (seq1.num() > 0) { + print(seq1, "seq1"); + } + if (seq2.num() > 0) { + print(seq2, "seq2"); + } + if (seq3.num() > 0) { + print(seq3, "seq3"); + } + } + + void print(HdrSeq& seq, const char* msg) { + std::cout << "["; + for (int i = 0; i <= 100; i += 10) { + std::cout << "\t" << seq.percentile(i); + } + std::cout << " ] : " << msg << "\n"; + } }; class BasicShenandoahNumberSeqTest: public ShenandoahNumberSeqTest { - protected: - const double err = 0.5; + public: BasicShenandoahNumberSeqTest() { - seq.add(0); - seq.add(1); - seq.add(10); + seq1.add(0); + seq1.add(1); + seq1.add(10); for (int i = 0; i < 7; i++) { - seq.add(100); + seq1.add(100); } - std::cout << " p0 = " << seq.percentile(0); - std::cout << " p10 = " << seq.percentile(10); - std::cout << " p20 = " << seq.percentile(20); - std::cout << " p30 = " << seq.percentile(30); - std::cout << " p50 = " << seq.percentile(50); - std::cout << " p80 = " << seq.percentile(80); - std::cout << " p90 = " << seq.percentile(90); - std::cout << " p100 = " << seq.percentile(100); + ShenandoahNumberSeqTest::print(); + } +}; + +class ShenandoahNumberSeqMergeTest: public ShenandoahNumberSeqTest { + public: + ShenandoahNumberSeqMergeTest() { + for (int i = 0; i < 80; i++) { + seq1.add(1); + seq3.add(1); + } + + for (int i = 0; i < 20; i++) { + seq2.add(100); + seq3.add(100); + } + ShenandoahNumberSeqTest::print(); } }; TEST_VM_F(BasicShenandoahNumberSeqTest, maximum_test) { - EXPECT_EQ(seq.maximum(), 100); + EXPECT_EQ(seq1.maximum(), 100); } TEST_VM_F(BasicShenandoahNumberSeqTest, minimum_test) { - EXPECT_EQ(0, seq.percentile(0)); + EXPECT_EQ(0, seq1.percentile(0)); } TEST_VM_F(BasicShenandoahNumberSeqTest, percentile_test) { - EXPECT_NEAR(0, seq.percentile(10), err); - EXPECT_NEAR(1, seq.percentile(20), err); - EXPECT_NEAR(10, seq.percentile(30), err); - EXPECT_NEAR(100, seq.percentile(40), err); - EXPECT_NEAR(100, seq.percentile(50), err); - EXPECT_NEAR(100, seq.percentile(75), err); - EXPECT_NEAR(100, seq.percentile(90), err); - EXPECT_NEAR(100, seq.percentile(100), err); + EXPECT_NEAR(0, seq1.percentile(10), err); + EXPECT_NEAR(1, seq1.percentile(20), err); + EXPECT_NEAR(10, seq1.percentile(30), err); + EXPECT_NEAR(100, seq1.percentile(40), err); + EXPECT_NEAR(100, seq1.percentile(50), err); + EXPECT_NEAR(100, seq1.percentile(75), err); + EXPECT_NEAR(100, seq1.percentile(90), err); + EXPECT_NEAR(100, seq1.percentile(100), err); +} + +TEST_VM_F(ShenandoahNumberSeqMergeTest, merge_test) { + EXPECT_EQ(seq1.num(), 80); + EXPECT_EQ(seq2.num(), 20); + EXPECT_FALSE(isnan(seq2.davg())); // Exercise the path; not a nan + EXPECT_FALSE(isnan(seq2.dsd())); + EXPECT_FALSE(isnan(seq2.dvariance())); + + std::cout << "Pre-merge: \n"; + print(); + seq1.merge(seq2); // clears seq1, after merging into seq2 + std::cout << "Post-merge: \n"; + print(); + + EXPECT_EQ(seq1.num(), 0); + EXPECT_EQ(seq2.num(), 100); + EXPECT_EQ(seq2.num(), seq3.num()); + EXPECT_TRUE(isnan(seq2.davg())); // until we fix decayed stats + EXPECT_TRUE(isnan(seq2.dvariance())); + + EXPECT_EQ(seq2.maximum(), seq3.maximum()); + EXPECT_EQ(seq2.percentile(0), seq3.percentile(0)); + for (int i = 0; i <= 100; i += 10) { + EXPECT_NEAR(seq2.percentile(i), seq3.percentile(i), err); + } + EXPECT_NEAR(seq2.avg(), seq3.avg(), err); + EXPECT_NEAR(seq2.sd(), seq3.sd(), err); } From 1a9623802c3595300fd796539e1a75aa3533cda6 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Mon, 19 Dec 2022 23:18:47 +0000 Subject: [PATCH 163/254] Use CardTable::card_size_in_words rather than hard coded constant Reviewed-by: ysr, kdnilsen --- .../gc/shenandoah/shenandoahScanRemembered.cpp | 15 +++++++++------ .../gc/shenandoah/shenandoahScanRemembered.hpp | 12 +++++------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp index 6d2d4c754656e..f05e9988a4bb1 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp @@ -139,7 +139,7 @@ size_t ShenandoahRegionChunkIterator::calc_num_groups() { size_t num_groups = 0; size_t cumulative_group_span = 0; size_t current_group_span = _first_group_chunk_size_b4_rebalance * _regular_group_size; - size_t smallest_group_span = _smallest_chunk_size_words * _regular_group_size; + size_t smallest_group_span = smallest_chunk_size_words() * _regular_group_size; while ((num_groups < _maximum_groups) && (cumulative_group_span + current_group_span <= total_heap_size)) { num_groups++; cumulative_group_span += current_group_span; @@ -179,7 +179,7 @@ size_t ShenandoahRegionChunkIterator::calc_total_chunks() { size_t num_chunks = 0; size_t cumulative_group_span = 0; size_t current_group_span = _first_group_chunk_size_b4_rebalance * _regular_group_size; - size_t smallest_group_span = _smallest_chunk_size_words * _regular_group_size; + size_t smallest_group_span = smallest_chunk_size_words() * _regular_group_size; // The first group gets special handling because the first chunk size can be no larger than _largest_chunk_size_words if (region_size_words > _maximum_chunk_size_words) { @@ -216,7 +216,7 @@ size_t ShenandoahRegionChunkIterator::calc_total_chunks() { } else if (current_group_span <= smallest_group_span) { // We cannot introduce new groups because we've reached the lower bound on group size. So this last // group may hold extra chunks. - size_t chunk_span = _smallest_chunk_size_words; + size_t chunk_span = smallest_chunk_size_words(); size_t extra_chunks = unspanned_heap_size / chunk_span; assert (extra_chunks * chunk_span == unspanned_heap_size, "Chunks must precisely span regions"); num_chunks += extra_chunks; @@ -249,11 +249,14 @@ ShenandoahRegionChunkIterator::ShenandoahRegionChunkIterator(ShenandoahHeap* hea _total_chunks(calc_total_chunks()), _index(0) { - assert(_smallest_chunk_size_words == _clusters_in_smallest_chunk * CardTable::card_size_in_words() - * ShenandoahCardCluster::CardsPerCluster, "_smallest_chunk_size is not valid"); +#ifdef ASSERT + size_t expected_chunk_size_words = _clusters_in_smallest_chunk * CardTable::card_size_in_words() * ShenandoahCardCluster::CardsPerCluster; + assert(smallest_chunk_size_words() == expected_chunk_size_words, "_smallest_chunk_size (" SIZE_FORMAT") is not valid because it does not equal (" SIZE_FORMAT ")", + smallest_chunk_size_words(), expected_chunk_size_words); +#endif assert(_num_groups <= _maximum_groups, "The number of remembered set scanning groups must be less than or equal to maximum groups"); - assert(_smallest_chunk_size_words << (_maximum_groups - 1) == _maximum_chunk_size_words, + assert(smallest_chunk_size_words() << (_maximum_groups - 1) == _maximum_chunk_size_words, "Maximum number of groups needs to span maximum chunk size to smallest chunk size"); size_t words_in_region = ShenandoahHeapRegion::region_size_words(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp index 053bb08d26261..d42e9913cb6f9 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp @@ -1024,16 +1024,14 @@ class ShenandoahRegionChunkIterator : public StackObj { static const size_t _maximum_chunk_size_words = (4 * 1024 * 1024) / HeapWordSize; static const size_t _clusters_in_smallest_chunk = 4; - static const size_t _assumed_words_in_card = 64; - // smallest_chunk_size is 4 clusters (i.e. 128 KiB). Note that there are 64 words per card and there are 64 cards per - // cluster. Each cluster spans 128 KiB. + // smallest_chunk_size is 4 clusters. Each cluster spans 128 KiB. // This is computed from CardTable::card_size_in_words() * // ShenandoahCardCluster::CardsPerCluster; - // We can't perform this computation here, because of encapsulation and initialization constraints. We paste - // the magic number here, and assert that this number matches the intended computation in constructor. - static const size_t _smallest_chunk_size_words = (_clusters_in_smallest_chunk * _assumed_words_in_card * - ShenandoahCardCluster::CardsPerCluster); + static size_t smallest_chunk_size_words() { + return _clusters_in_smallest_chunk * CardTable::card_size_in_words() * + ShenandoahCardCluster::CardsPerCluster; + } // The total remembered set scanning effort is divided into chunks of work that are assigned to individual worker tasks. // The chunks of assigned work are divided into groups, where the size of the typical group (_regular_group_size) is half the From 9114616c01bdeeddad50bec93869decee90f5a58 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Tue, 20 Dec 2022 21:45:15 +0000 Subject: [PATCH 164/254] Shrink tlab to capacity Reviewed-by: ysr, wkemper --- .../share/gc/shenandoah/shenandoahFreeSet.cpp | 1 - .../share/gc/shenandoah/shenandoahHeap.cpp | 276 +++++++++++------- 2 files changed, 163 insertions(+), 114 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index deee5fed0e0e8..401611692560e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -133,7 +133,6 @@ HeapWord* ShenandoahFreeSet::allocate_single(ShenandoahAllocRequest& req, bool& } } } - // There is no recovery. Mutator does not touch collector view at all. break; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 15fac82f2d10a..90428277d95b7 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -862,7 +862,7 @@ void ShenandoahHeap::report_promotion_failure(Thread* thread, size_t size) { size, plab == nullptr? "no": "yes", words_remaining, promote_enabled, promotion_reserve, promotion_expended); if ((gc_id == last_report_epoch) && (epoch_report_count >= MaxReportsPerEpoch)) { - log_info(gc, ergo)("Squelching additional promotion failure reports for epoch " SIZE_FORMAT, last_report_epoch); + log_info(gc, ergo)("Squelching additional promotion failure reports for current epoch"); } else if (gc_id != last_report_epoch) { last_report_epoch = gc_id;; epoch_report_count = 1; @@ -1102,6 +1102,7 @@ bool ShenandoahHeap::adjust_generation_sizes() { return false; } +// Called from stubs in JIT code or interpreter HeapWord* ShenandoahHeap::allocate_new_tlab(size_t min_size, size_t requested_size, size_t* actual_size) { @@ -1169,21 +1170,17 @@ HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req, bool is_p // // Then, we need to make sure the allocation was retried after at least one // Full GC, which means we want to try more than ShenandoahFullGCThreshold times. - size_t tries = 0; - while (result == NULL && _progress_last_gc.is_set()) { tries++; control_thread()->handle_alloc_failure(req); result = allocate_memory_under_lock(req, in_new_region, is_promotion); } - while (result == NULL && tries <= ShenandoahFullGCThreshold) { tries++; control_thread()->handle_alloc_failure(req); result = allocate_memory_under_lock(req, in_new_region, is_promotion); } - } else { assert(req.is_gc_alloc(), "Can only accept GC allocs here"); result = allocate_memory_under_lock(req, in_new_region, is_promotion); @@ -1225,120 +1222,169 @@ HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req, bool is_p } HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req, bool& in_new_region, bool is_promotion) { - // promotion_eligible pertains only to PLAB allocations, denoting that the PLAB is allowed to allocate for promotions. - bool promotion_eligible = false; - bool allow_allocation = true; - bool plab_alloc = false; - size_t requested_bytes = req.size() * HeapWordSize; - HeapWord* result = nullptr; - ShenandoahHeapLocker locker(lock()); - Thread* thread = Thread::current(); - if (mode()->is_generational()) { - if (req.affiliation() == YOUNG_GENERATION) { - if (req.is_mutator_alloc()) { - if (requested_bytes >= young_generation()->adjusted_available()) { - // We know this is not a GCLAB. This must be a TLAB or a shared allocation. Reject the allocation request if - // exceeds established capacity limits. - - // TODO: if ShenandoahElasticTLAB and req.is_lab_alloc(), we should endeavor to shrink the TLAB request - // in order to avoid allocation failure and degeneration of GC. - - log_info(gc, ergo)("Rejecting mutator alloc of " SIZE_FORMAT " because young available is: " SIZE_FORMAT, - requested_bytes, young_generation()->adjusted_available()); - return nullptr; - } - } - } else { // reg.affiliation() == OLD_GENERATION - assert(req.type() != ShenandoahAllocRequest::_alloc_gclab, "GCLAB pertains only to young-gen memory"); - if (req.type() == ShenandoahAllocRequest::_alloc_plab) { - plab_alloc = true; - size_t promotion_avail = get_promoted_reserve(); - size_t promotion_expended = get_promoted_expended(); - if (promotion_expended + requested_bytes > promotion_avail) { - promotion_avail = 0; - if (get_old_evac_reserve() == 0) { - // There are no old-gen evacuations in this pass. There's no value in creating a plab that cannot - // be used for promotions. - allow_allocation = false; - } - } else { - promotion_avail = promotion_avail - (promotion_expended + requested_bytes); - promotion_eligible = true; - } - } else if (is_promotion) { - // This is a shared alloc for promotion - size_t promotion_avail = get_promoted_reserve(); - size_t promotion_expended = get_promoted_expended(); - if (promotion_expended + requested_bytes > promotion_avail) { - promotion_avail = 0; - } else { - promotion_avail = promotion_avail - (promotion_expended + requested_bytes); - } + bool try_smaller_lab_size = false; + size_t smaller_lab_size; + { + // promotion_eligible pertains only to PLAB allocations, denoting that the PLAB is allowed to allocate for promotions. + bool promotion_eligible = false; + bool allow_allocation = true; + bool plab_alloc = false; + size_t requested_bytes = req.size() * HeapWordSize; + HeapWord* result = nullptr; + ShenandoahHeapLocker locker(lock()); + Thread* thread = Thread::current(); - if (promotion_avail == 0) { - // We need to reserve the remaining memory for evacuation. Reject this allocation. The object will be - // evacuated to young-gen memory and promoted during a future GC pass. - return nullptr; + if (mode()->is_generational()) { + if (req.affiliation() == YOUNG_GENERATION) { + if (req.is_mutator_alloc()) { + size_t young_available = young_generation()->adjusted_available(); + if (requested_bytes > young_available) { + // We know this is not a GCLAB. This must be a TLAB or a shared allocation. + if (req.is_lab_alloc() && (young_available >= req.min_size())) { + try_smaller_lab_size = true; + smaller_lab_size = young_available / HeapWordSize; + } else { + // Can't allocate because even min_size() is larger than remaining young_available + log_info(gc, ergo)("Unable to shrink %s alloc request of minimum size: " SIZE_FORMAT + ", young available: " SIZE_FORMAT, + req.is_lab_alloc()? "TLAB": "shared", + HeapWordSize * (req.is_lab_alloc()? req.min_size(): req.size()), young_available); + return nullptr; + } + } } - // Else, we'll allow the allocation to proceed. (Since we hold heap lock, the tested condition remains true.) - } else { - // This is a shared allocation for evacuation. Memory has already been reserved for this purpose. - } - } - } - result = (allow_allocation)? _free_set->allocate(req, in_new_region): nullptr; - if (result != NULL) { - if (req.affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) { - ShenandoahThreadLocalData::reset_plab_promoted(thread); - if (req.is_gc_alloc()) { + } else { // reg.affiliation() == OLD_GENERATION + assert(req.type() != ShenandoahAllocRequest::_alloc_gclab, "GCLAB pertains only to young-gen memory"); if (req.type() == ShenandoahAllocRequest::_alloc_plab) { - if (promotion_eligible) { - size_t actual_size = req.actual_size() * HeapWordSize; - // Assume the entirety of this PLAB will be used for promotion. This prevents promotion from overreach. - // When we retire this plab, we'll unexpend what we don't really use. - ShenandoahThreadLocalData::enable_plab_promotions(thread); - expend_promoted(actual_size); - assert(get_promoted_expended() <= get_promoted_reserve(), "Do not expend more promotion than budgeted"); - ShenandoahThreadLocalData::set_plab_preallocated_promoted(thread, actual_size); + plab_alloc = true; + size_t promotion_avail = get_promoted_reserve(); + size_t promotion_expended = get_promoted_expended(); + if (promotion_expended + requested_bytes > promotion_avail) { + promotion_avail = 0; + if (get_old_evac_reserve() == 0) { + // There are no old-gen evacuations in this pass. There's no value in creating a plab that cannot + // be used for promotions. + allow_allocation = false; + } } else { - // Disable promotions in this thread because entirety of this PLAB must be available to hold old-gen evacuations. - ShenandoahThreadLocalData::disable_plab_promotions(thread); - ShenandoahThreadLocalData::set_plab_preallocated_promoted(thread, 0); + promotion_avail = promotion_avail - (promotion_expended + requested_bytes); + promotion_eligible = true; } } else if (is_promotion) { - // Shared promotion. Assume size is requested_bytes. - expend_promoted(requested_bytes); - assert(get_promoted_expended() <= get_promoted_reserve(), "Do not expend more promotion than budgeted"); + // This is a shared alloc for promotion + size_t promotion_avail = get_promoted_reserve(); + size_t promotion_expended = get_promoted_expended(); + if (promotion_expended + requested_bytes > promotion_avail) { + promotion_avail = 0; + } else { + promotion_avail = promotion_avail - (promotion_expended + requested_bytes); + } + if (promotion_avail == 0) { + // We need to reserve the remaining memory for evacuation. Reject this allocation. The object will be + // evacuated to young-gen memory and promoted during a future GC pass. + return nullptr; + } + // Else, we'll allow the allocation to proceed. (Since we hold heap lock, the tested condition remains true.) + } else { + // This is a shared allocation for evacuation. Memory has already been reserved for this purpose. } } + } // This ends the is_generational() block + + if (!try_smaller_lab_size) { + result = (allow_allocation)? _free_set->allocate(req, in_new_region): nullptr; + if (result != NULL) { + if (req.affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) { + ShenandoahThreadLocalData::reset_plab_promoted(thread); + if (req.is_gc_alloc()) { + if (req.type() == ShenandoahAllocRequest::_alloc_plab) { + if (promotion_eligible) { + size_t actual_size = req.actual_size() * HeapWordSize; + // Assume the entirety of this PLAB will be used for promotion. This prevents promotion from overreach. + // When we retire this plab, we'll unexpend what we don't really use. + ShenandoahThreadLocalData::enable_plab_promotions(thread); + expend_promoted(actual_size); + assert(get_promoted_expended() <= get_promoted_reserve(), "Do not expend more promotion than budgeted"); + ShenandoahThreadLocalData::set_plab_preallocated_promoted(thread, actual_size); + } else { + // Disable promotions in this thread because entirety of this PLAB must be available to hold old-gen evacuations. + ShenandoahThreadLocalData::disable_plab_promotions(thread); + ShenandoahThreadLocalData::set_plab_preallocated_promoted(thread, 0); + } + } else if (is_promotion) { + // Shared promotion. Assume size is requested_bytes. + expend_promoted(requested_bytes); + assert(get_promoted_expended() <= get_promoted_reserve(), "Do not expend more promotion than budgeted"); + } + } - // Register the newly allocated object while we're holding the global lock since there's no synchronization - // built in to the implementation of register_object(). There are potential races when multiple independent - // threads are allocating objects, some of which might span the same card region. For example, consider - // a card table's memory region within which three objects are being allocated by three different threads: - // - // objects being "concurrently" allocated: - // [-----a------][-----b-----][--------------c------------------] - // [---- card table memory range --------------] - // - // Before any objects are allocated, this card's memory range holds no objects. Note that: - // allocation of object a wants to set the has-object, first-start, and last-start attributes of the preceding card region. - // allocation of object b wants to set the has-object, first-start, and last-start attributes of this card region. - // allocation of object c also wants to set the has-object, first-start, and last-start attributes of this card region. - // - // The thread allocating b and the thread allocating c can "race" in various ways, resulting in confusion, such as last-start - // representing object b while first-start represents object c. This is why we need to require all register_object() - // invocations to be "mutually exclusive" with respect to each card's memory range. - ShenandoahHeap::heap()->card_scan()->register_object(result); - } - } else { - // The allocation failed. If this was a plab allocation, We've already retired it and no longer have a plab. - if ((req.affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) && req.is_gc_alloc() && - (req.type() == ShenandoahAllocRequest::_alloc_plab)) { - // We don't need to disable PLAB promotions because there is no PLAB. We leave promotions enabled because - // this allows the surrounding infrastructure to retry alloc_plab_slow() with a smaller PLAB size. - ShenandoahThreadLocalData::set_plab_preallocated_promoted(thread, 0); - } + // Register the newly allocated object while we're holding the global lock since there's no synchronization + // built in to the implementation of register_object(). There are potential races when multiple independent + // threads are allocating objects, some of which might span the same card region. For example, consider + // a card table's memory region within which three objects are being allocated by three different threads: + // + // objects being "concurrently" allocated: + // [-----a------][-----b-----][--------------c------------------] + // [---- card table memory range --------------] + // + // Before any objects are allocated, this card's memory range holds no objects. Note that allocation of object a + // wants to set the has-object, first-start, and last-start attributes of the preceding card region. + // allocation of object b wants to set the has-object, first-start, and last-start attributes of this card region. + // allocation of object c also wants to set the has-object, first-start, and last-start attributes of this card region. + // + // The thread allocating b and the thread allocating c can "race" in various ways, resulting in confusion, such as + // last-start representing object b while first-start represents object c. This is why we need to require all + // register_object() invocations to be "mutually exclusive" with respect to each card's memory range. + ShenandoahHeap::heap()->card_scan()->register_object(result); + } + } else { + // The allocation failed. If this was a plab allocation, We've already retired it and no longer have a plab. + if ((req.affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) && req.is_gc_alloc() && + (req.type() == ShenandoahAllocRequest::_alloc_plab)) { + // We don't need to disable PLAB promotions because there is no PLAB. We leave promotions enabled because + // this allows the surrounding infrastructure to retry alloc_plab_slow() with a smaller PLAB size. + ShenandoahThreadLocalData::set_plab_preallocated_promoted(thread, 0); + } + } + return result; + } + // else, try_smaller_lab_size is true so we fall through and recurse with a smaller lab size + } // This closes the block that holds the heap lock. This releases the lock. + + // We arrive here if the tlab allocation request can be resized to fit within young_available + assert((req.affiliation() == YOUNG_GENERATION) && req.is_lab_alloc() && req.is_mutator_alloc() && + (smaller_lab_size < req.size()), "Only shrink allocation request size for TLAB allocations"); + + // By convention, ShenandoahAllocationRequest is primarily read-only. The only mutable instance data is represented by + // actual_size(), which is overwritten with the size of the allocaion when the allocation request is satisfied. We use a + // recursive call here rather than introducing new methods to mutate the existing ShenandoahAllocationRequest argument. + // Mutation of the existing object might result in astonishing results if calling contexts assume the content of immutable + // fields remain constant. The original TLAB allocation request was for memory that exceeded the current capacity. We'll + // attempt to allocate a smaller TLAB. If this is successful, we'll update actual_size() of our incoming + // ShenandoahAllocRequest. If the recursive request fails, we'll simply return nullptr. + + // Note that we've relinquished the HeapLock and some other thread may perform additional allocation before our recursive + // call reacquires the lock. If that happens, we will need another recursive call to further reduce the size of our request + // for each time another thread allocates young memory during the brief intervals that the heap lock is available to + // interfering threads. We expect this interference to be rare. The recursion bottoms out when young_available is + // smaller than req.min_size(). The inner-nested call to allocate_memory_under_lock() uses the same min_size() value + // as this call, but it uses a preferred size() that is smaller than our preferred size, and is no larger than what we most + // recently saw as the memory currently available within the young generation. + + // TODO: At the expense of code clarity, we could rewrite this recursive solution to use iteration. We need at most one + // extra instance of the ShenandoahAllocRequest, which we can re-initialize multiple times inside a loop, with one iteration + // of the loop required for each time the existing solution would recurse. An iterative solution would be more efficient + // in CPU time and stack memory utilization. The expectation is that it is very rare that we would recurse more than once + // so making this change is not currently seen as a high priority. + + ShenandoahAllocRequest smaller_req = ShenandoahAllocRequest::for_tlab(req.min_size(), smaller_lab_size); + + // Note that shrinking the preferred size gets us past the gatekeeper that checks whether there's available memory to + // satisfy the allocation request. The reality is the actual TLAB size is likely to be even smaller, because it will + // depend on how much memory is available within mutator regions that are not yet fully used. + HeapWord* result = allocate_memory_under_lock(smaller_req, in_new_region, is_promotion); + if (result != nullptr) { + req.set_actual_size(smaller_req.actual_size()); } return result; } @@ -1704,9 +1750,13 @@ void ShenandoahHeap::set_young_lab_region_flags() { // Returns size in bytes size_t ShenandoahHeap::unsafe_max_tlab_alloc(Thread *thread) const { if (ShenandoahElasticTLAB) { - // With Elastic TLABs, return the max allowed size, and let the allocation path - // figure out the safe size for current allocation. - return ShenandoahHeapRegion::max_tlab_size_bytes(); + if (mode()->is_generational()) { + return MIN2(ShenandoahHeapRegion::max_tlab_size_bytes(), young_generation()->adjusted_available()); + } else { + // With Elastic TLABs, return the max allowed size, and let the allocation path + // figure out the safe size for current allocation. + return ShenandoahHeapRegion::max_tlab_size_bytes(); + } } else { return MIN2(_free_set->unsafe_peek_free(), ShenandoahHeapRegion::max_tlab_size_bytes()); } From da9501170820b3e32b903228bd921aaa860e90c0 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Wed, 21 Dec 2022 21:52:44 +0000 Subject: [PATCH 165/254] Initial sizing refactor Reviewed-by: kdnilsen --- .../shenandoahAdaptiveHeuristics.cpp | 4 - .../mode/shenandoahGenerationalMode.cpp | 1 - .../share/gc/shenandoah/shenandoahAsserts.hpp | 3 + .../gc/shenandoah/shenandoahGeneration.cpp | 2 + .../gc/shenandoah/shenandoahGeneration.hpp | 9 +- .../share/gc/shenandoah/shenandoahHeap.cpp | 80 +++--- .../share/gc/shenandoah/shenandoahHeap.hpp | 8 +- .../gc/shenandoah/shenandoahInitLogger.cpp | 11 +- .../gc/shenandoah/shenandoahMmuTracker.cpp | 230 +++++++++++++----- .../gc/shenandoah/shenandoahMmuTracker.hpp | 86 +++++-- .../shenandoah/shenandoahYoungGeneration.cpp | 6 +- .../shenandoah/shenandoahYoungGeneration.hpp | 6 + .../test_shenandoahOldHeuristic.cpp | 2 + 13 files changed, 319 insertions(+), 129 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp index dbcc7eabd8de9..a22d64c655628 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp @@ -317,10 +317,6 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { ", max_capacity: " SIZE_FORMAT ", allocated: " SIZE_FORMAT, _generation->name(), available, capacity, max_capacity, allocated); - // Make sure the code below treats available without the soft tail. - size_t soft_tail = max_capacity - capacity; - available = (available > soft_tail) ? (available - soft_tail) : 0; - // The collector reserve may eat into what the mutator is allowed to use. Make sure we are looking // at what is available to the mutator when deciding whether to start a GC. size_t usable = ShenandoahHeap::heap()->free_set()->available(); diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp index 1600b187e81ac..0e83ef59935b0 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp @@ -37,7 +37,6 @@ void ShenandoahGenerationalMode::initialize_flags() const { } SHENANDOAH_ERGO_OVERRIDE_DEFAULT(GCTimeRatio, 70); - SHENANDOAH_ERGO_OVERRIDE_DEFAULT(NewRatio, 1); SHENANDOAH_ERGO_OVERRIDE_DEFAULT(ShenandoahUnloadClassesFrequency, 0); SHENANDOAH_ERGO_ENABLE_FLAG(ExplicitGCInvokesConcurrent); SHENANDOAH_ERGO_ENABLE_FLAG(ShenandoahImplicitGCInvokesConcurrent); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahAsserts.hpp b/src/hotspot/share/gc/shenandoah/shenandoahAsserts.hpp index 86a3cf891615d..778d42a7fdedc 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahAsserts.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahAsserts.hpp @@ -167,6 +167,8 @@ class ShenandoahAsserts { #define shenandoah_assert_heaplocked_or_fullgc_safepoint() \ ShenandoahAsserts::assert_heaplocked_or_fullgc_safepoint(__FILE__, __LINE__) +#define shenandoah_assert_control_or_vm_thread() \ + assert(Thread::current()->is_VM_thread() || Thread::current() == ShenandoahHeap::heap()->control_thread(), "Expected control thread or vm thread") #define shenandoah_assert_generational() \ assert(ShenandoahHeap::heap()->mode()->is_generational(), "Must be in generational mode here.") @@ -221,6 +223,7 @@ class ShenandoahAsserts { #define shenandoah_assert_not_heaplocked() #define shenandoah_assert_heaplocked_or_safepoint() #define shenandoah_assert_heaplocked_or_fullgc_safepoint() +#define shenandoah_assert_control_or_vm_thread() #define shenandoah_assert_generational() #endif diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index 7eab9636f7544..cb3b2c0a7358c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -991,12 +991,14 @@ size_t ShenandoahGeneration::adjusted_unaffiliated_regions() const { void ShenandoahGeneration::increase_capacity(size_t increment) { shenandoah_assert_heaplocked_or_safepoint(); + assert(_max_capacity + increment <= ShenandoahHeap::heap()->max_size_for(this), "Cannot increase generation capacity beyond maximum."); _max_capacity += increment; _soft_max_capacity += increment; } void ShenandoahGeneration::decrease_capacity(size_t decrement) { shenandoah_assert_heaplocked_or_safepoint(); + assert(_max_capacity - decrement >= ShenandoahHeap::heap()->min_size_for(this), "Cannot decrease generation capacity beyond minimum."); _max_capacity -= decrement; _soft_max_capacity -= decrement; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index d915149bb54ff..52882ba84196b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -168,7 +168,7 @@ class ShenandoahGeneration : public CHeapObj { virtual void reserve_task_queues(uint workers); virtual ShenandoahObjToScanQueueSet* old_gen_task_queues() const; - // Scan remembered set at start of concurrent young-gen marking. */ + // Scan remembered set at start of concurrent young-gen marking. void scan_remembered_set(bool is_concurrent); // Return the updated value of affiliated_region_count @@ -187,7 +187,14 @@ class ShenandoahGeneration : public CHeapObj { virtual void record_success_concurrent(bool abbreviated); virtual void record_success_degenerated(); + // Record the total on-cpu time a thread has spent collecting this + // generation. This is only called by the control thread (at the start + // of a collection) and by the VM thread at the end of the collection, + // so there are no locking concerns. virtual void add_collection_time(double time_seconds); + + // This returns the accumulated collection time and resets it to zero. + // This is used to decide which generation should be resized. double reset_collection_time(); }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 90428277d95b7..e1841d986c2dd 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -447,16 +447,42 @@ jint ShenandoahHeap::initialize() { return JNI_OK; } +size_t ShenandoahHeap::max_size_for(ShenandoahGeneration* generation) const { + switch (generation->generation_mode()) { + case YOUNG: return _generation_sizer.max_young_size(); + case OLD: return max_capacity() - _generation_sizer.min_young_size(); + case GLOBAL: return max_capacity(); + default: + ShouldNotReachHere(); + return 0; + } +} + +size_t ShenandoahHeap::min_size_for(ShenandoahGeneration* generation) const { + switch (generation->generation_mode()) { + case YOUNG: return _generation_sizer.min_young_size(); + case OLD: return max_capacity() - _generation_sizer.max_young_size(); + case GLOBAL: return min_capacity(); + default: + ShouldNotReachHere(); + return 0; + } +} + void ShenandoahHeap::initialize_generations() { - size_t max_capacity_new = young_generation_capacity(max_capacity()); - size_t soft_max_capacity_new = young_generation_capacity(soft_max_capacity()); - size_t max_capacity_old = max_capacity() - max_capacity_new; - size_t soft_max_capacity_old = soft_max_capacity() - soft_max_capacity_new; + // Max capacity is the maximum _allowed_ capacity. That is, the maximum allowed capacity + // for old would be total heap - minimum capacity of young. This means the sum of the maximum + // allowed for old and young could exceed the total heap size. It remains the case that the + // _actual_ capacity of young + old = total. + _generation_sizer.heap_size_changed(soft_max_capacity()); + size_t initial_capacity_young = _generation_sizer.max_young_size(); + size_t max_capacity_young = _generation_sizer.max_young_size(); + size_t initial_capacity_old = max_capacity() - max_capacity_young; + size_t max_capacity_old = max_capacity() - initial_capacity_young; - _young_generation = new ShenandoahYoungGeneration(_max_workers, max_capacity_new, soft_max_capacity_new); - _old_generation = new ShenandoahOldGeneration(_max_workers, max_capacity_old, soft_max_capacity_old); - _global_generation = new ShenandoahGlobalGeneration(_max_workers, max_capacity_new + max_capacity_old, - soft_max_capacity_new + soft_max_capacity_old); + _young_generation = new ShenandoahYoungGeneration(_max_workers, max_capacity_young, initial_capacity_young); + _old_generation = new ShenandoahOldGeneration(_max_workers, max_capacity_old, initial_capacity_old); + _global_generation = new ShenandoahGlobalGeneration(_max_workers, soft_max_capacity(), soft_max_capacity()); } void ShenandoahHeap::initialize_heuristics() { @@ -535,6 +561,8 @@ ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) : _verifier(NULL), _phase_timings(NULL), _evac_tracker(new ShenandoahEvacuationTracker()), + _mmu_tracker(), + _generation_sizer(&_mmu_tracker), _monitoring_support(NULL), _memory_pool(NULL), _young_gen_memory_pool(NULL), @@ -662,37 +690,6 @@ bool ShenandoahHeap::is_gc_generation_young() const { return _gc_generation != NULL && _gc_generation->generation_mode() == YOUNG; } -// There are three JVM parameters for setting young gen capacity: -// NewSize, MaxNewSize, NewRatio. -// -// If only NewSize is set, it assigns a fixed size and the other two parameters are ignored. -// Otherwise NewRatio applies. -// -// If NewSize is set in any combination, it provides a lower bound. -// -// If MaxNewSize is set it provides an upper bound. -// If this bound is smaller than NewSize, it supersedes, -// resulting in a fixed size given by MaxNewSize. -size_t ShenandoahHeap::young_generation_capacity(size_t capacity) { - if (strcmp(ShenandoahGCMode, "generational") == 0) { - if (FLAG_IS_CMDLINE(NewSize) && !FLAG_IS_CMDLINE(MaxNewSize) && !FLAG_IS_CMDLINE(NewRatio)) { - capacity = MIN2(NewSize, capacity); - } else { - capacity /= NewRatio + 1; - if (FLAG_IS_CMDLINE(NewSize)) { - capacity = MAX2(NewSize, capacity); - } - if (FLAG_IS_CMDLINE(MaxNewSize)) { - capacity = MIN2(MaxNewSize, capacity); - } - } - // capacity must be a multiple of ShenandoahHeapRegion::region_size_bytes() - capacity &= ~ShenandoahHeapRegion::region_size_bytes_mask(); - } - // else, make no adjustment to global capacity - return capacity; -} - size_t ShenandoahHeap::used() const { return Atomic::load(&_used); } @@ -761,7 +758,8 @@ void ShenandoahHeap::set_soft_max_capacity(size_t v) { Atomic::store(&_soft_max_size, v); if (mode()->is_generational()) { - size_t soft_max_capacity_young = young_generation_capacity(_soft_max_size); + _generation_sizer.heap_size_changed(_soft_max_size); + size_t soft_max_capacity_young = _generation_sizer.max_young_size(); size_t soft_max_capacity_old = _soft_max_size - soft_max_capacity_young; _young_generation->set_soft_max_capacity(soft_max_capacity_young); _old_generation->set_soft_max_capacity(soft_max_capacity_old); @@ -1097,7 +1095,7 @@ void ShenandoahHeap::coalesce_and_fill_old_regions() { bool ShenandoahHeap::adjust_generation_sizes() { if (mode()->is_generational()) { - return _mmu_tracker.adjust_generation_sizes(); + return _generation_sizer.adjust_generation_sizes(); } return false; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index 239469ea41dd4..751b57f72025e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -223,7 +223,6 @@ class ShenandoahHeap : public CollectedHeap { volatile size_t _committed; shenandoah_padding(1); - static size_t young_generation_capacity(size_t total_capacity); void help_verify_region_rem_set(ShenandoahHeapRegion* r, ShenandoahMarkingContext* ctx, HeapWord* from, HeapWord* top, HeapWord* update_watermark, const char* message); @@ -537,15 +536,20 @@ class ShenandoahHeap : public CollectedHeap { ShenandoahPhaseTimings* _phase_timings; ShenandoahEvacuationTracker* _evac_tracker; ShenandoahMmuTracker _mmu_tracker; + ShenandoahGenerationSizer _generation_sizer; - ShenandoahControlThread* control_thread() { return _control_thread; } ShenandoahRegulatorThread* regulator_thread() { return _regulator_thread; } public: + ShenandoahControlThread* control_thread() { return _control_thread; } ShenandoahYoungGeneration* young_generation() const { return _young_generation; } ShenandoahGeneration* global_generation() const { return _global_generation; } ShenandoahOldGeneration* old_generation() const { return _old_generation; } ShenandoahGeneration* generation_for(ShenandoahRegionAffiliation affiliation) const; + const ShenandoahGenerationSizer* generation_sizer() const { return &_generation_sizer; } + + size_t max_size_for(ShenandoahGeneration* generation) const; + size_t min_size_for(ShenandoahGeneration* generation) const; ShenandoahCollectorPolicy* shenandoah_policy() const { return _shenandoah_policy; } ShenandoahMode* mode() const { return _gc_mode; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahInitLogger.cpp b/src/hotspot/share/gc/shenandoah/shenandoahInitLogger.cpp index b7acc53307768..dd3176520a901 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahInitLogger.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahInitLogger.cpp @@ -46,10 +46,19 @@ void ShenandoahInitLogger::print_heap() { log_info(gc, init)("Heuristics: %s", heap->global_generation()->heuristics()->name()); } else { log_info(gc, init)("Young Heuristics: %s", heap->young_generation()->heuristics()->name()); - log_info(gc, init)("Old Heuristics: %s", heap->old_generation()->heuristics()->name()); + log_info(gc, init)("Young Generation Initial Size: " SIZE_FORMAT "%s", + byte_size_in_proper_unit(heap->young_generation()->soft_max_capacity()), + proper_unit_for_byte_size(heap->young_generation()->soft_max_capacity())); log_info(gc, init)("Young Generation Max: " SIZE_FORMAT "%s", byte_size_in_proper_unit(heap->young_generation()->max_capacity()), proper_unit_for_byte_size(heap->young_generation()->max_capacity())); + log_info(gc, init)("Old Heuristics: %s", heap->old_generation()->heuristics()->name()); + log_info(gc, init)("Old Generation Initial Size: " SIZE_FORMAT "%s", + byte_size_in_proper_unit(heap->old_generation()->soft_max_capacity()), + proper_unit_for_byte_size(heap->old_generation()->soft_max_capacity())); + log_info(gc, init)("Old Generation Max: " SIZE_FORMAT "%s", + byte_size_in_proper_unit(heap->old_generation()->max_capacity()), + proper_unit_for_byte_size(heap->old_generation()->max_capacity())); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp index 4043145e07ff6..d981efba0bde7 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp @@ -23,6 +23,7 @@ */ #include "precompiled.hpp" +#include "gc/shenandoah/shenandoahAsserts.hpp" #include "gc/shenandoah/shenandoahMmuTracker.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahOldGeneration.hpp" @@ -54,9 +55,9 @@ class ThreadTimeAccumulator : public ThreadClosure { double ShenandoahMmuTracker::gc_thread_time_seconds() { ThreadTimeAccumulator cl; + // We include only the gc threads because those are the only threads + // we are responsible for. ShenandoahHeap::heap()->gc_threads_do(&cl); - // Include VM thread? Compiler threads? or no - because there - // is nothing the collector can do about those threads. return double(cl.total_time) / NANOSECS_PER_SEC; } @@ -70,11 +71,10 @@ double ShenandoahMmuTracker::process_time_seconds() { } ShenandoahMmuTracker::ShenandoahMmuTracker() : - _initial_collector_time_s(0.0), - _initial_process_time_s(0.0), - _resize_increment(YoungGenerationSizeIncrement / 100.0), - _mmu_periodic_task(new ShenandoahMmuTask(this)), - _mmu_average(10, ShenandoahAdaptiveDecayFactor) { + _generational_reference_time_s(0.0), + _process_reference_time_s(0.0), + _mmu_periodic_task(new ShenandoahMmuTask(this)), + _mmu_average(10, ShenandoahAdaptiveDecayFactor) { } ShenandoahMmuTracker::~ShenandoahMmuTracker() { @@ -83,56 +83,157 @@ ShenandoahMmuTracker::~ShenandoahMmuTracker() { } void ShenandoahMmuTracker::record(ShenandoahGeneration* generation) { - // This is only called by the control thread. + shenandoah_assert_control_or_vm_thread(); double collector_time_s = gc_thread_time_seconds(); - double elapsed_gc_time_s = collector_time_s - _initial_collector_time_s; + double elapsed_gc_time_s = collector_time_s - _generational_reference_time_s; generation->add_collection_time(elapsed_gc_time_s); - _initial_collector_time_s = collector_time_s; + _generational_reference_time_s = collector_time_s; } void ShenandoahMmuTracker::report() { // This is only called by the periodic thread. double process_time_s = process_time_seconds(); - double elapsed_process_time_s = process_time_s - _initial_process_time_s; - _initial_process_time_s = process_time_s; + double elapsed_process_time_s = process_time_s - _process_reference_time_s; + _process_reference_time_s = process_time_s; double verify_time_s = gc_thread_time_seconds(); - double verify_elapsed = verify_time_s - _initial_verify_collector_time_s; - _initial_verify_collector_time_s = verify_time_s; + double verify_elapsed = verify_time_s - _collector_reference_time_s; + _collector_reference_time_s = verify_time_s; double verify_mmu = ((elapsed_process_time_s - verify_elapsed) / elapsed_process_time_s) * 100; _mmu_average.add(verify_mmu); log_info(gc)("Average MMU = %.3f", _mmu_average.davg()); } -bool ShenandoahMmuTracker::adjust_generation_sizes() { +void ShenandoahMmuTracker::initialize() { + _process_reference_time_s = process_time_seconds(); + _generational_reference_time_s = gc_thread_time_seconds(); + _collector_reference_time_s = _generational_reference_time_s; + _mmu_periodic_task->enroll(); +} + +ShenandoahGenerationSizer::ShenandoahGenerationSizer(ShenandoahMmuTracker* mmu_tracker) + : _sizer_kind(SizerDefaults), + _use_adaptive_sizing(true), + _min_desired_young_size(0), + _max_desired_young_size(0), + _resize_increment(YoungGenerationSizeIncrement / 100.0), + _mmu_tracker(mmu_tracker) { + + if (FLAG_IS_CMDLINE(NewRatio)) { + if (FLAG_IS_CMDLINE(NewSize) || FLAG_IS_CMDLINE(MaxNewSize)) { + log_warning(gc, ergo)("-XX:NewSize and -XX:MaxNewSize override -XX:NewRatio"); + } else { + _sizer_kind = SizerNewRatio; + _use_adaptive_sizing = false; + return; + } + } + + if (NewSize > MaxNewSize) { + if (FLAG_IS_CMDLINE(MaxNewSize)) { + log_warning(gc, ergo)("NewSize (" SIZE_FORMAT "k) is greater than the MaxNewSize (" SIZE_FORMAT "k). " + "A new max generation size of " SIZE_FORMAT "k will be used.", + NewSize/K, MaxNewSize/K, NewSize/K); + } + FLAG_SET_ERGO(MaxNewSize, NewSize); + } + + if (FLAG_IS_CMDLINE(NewSize)) { + _min_desired_young_size = MAX2(NewSize, ShenandoahHeapRegion::region_size_bytes()); + if (FLAG_IS_CMDLINE(MaxNewSize)) { + _max_desired_young_size = MAX2(MaxNewSize, ShenandoahHeapRegion::region_size_bytes()); + _sizer_kind = SizerMaxAndNewSize; + _use_adaptive_sizing = _min_desired_young_size != _max_desired_young_size; + } else { + _sizer_kind = SizerNewSizeOnly; + } + } else if (FLAG_IS_CMDLINE(MaxNewSize)) { + _max_desired_young_size = MAX2(MaxNewSize, ShenandoahHeapRegion::region_size_bytes()); + _sizer_kind = SizerMaxNewSizeOnly; + } +} + +size_t ShenandoahGenerationSizer::calculate_min_size(size_t heap_size) { + size_t default_value = (heap_size * ShenandoahMinYoungPercentage) / 100; + return MAX2(ShenandoahHeapRegion::region_size_bytes(), default_value); +} + +size_t ShenandoahGenerationSizer::calculate_max_size(size_t heap_size) { + size_t default_value = (heap_size * ShenandoahMaxYoungPercentage) / 100; + return MAX2(ShenandoahHeapRegion::region_size_bytes(), default_value); +} + +void ShenandoahGenerationSizer::recalculate_min_max_young_length(size_t heap_size) { + assert(heap_size > 0, "Heap must be initialized"); + + switch (_sizer_kind) { + case SizerDefaults: + _min_desired_young_size = calculate_min_size(heap_size); + _max_desired_young_size = calculate_max_size(heap_size); + break; + case SizerNewSizeOnly: + _max_desired_young_size = calculate_max_size(heap_size); + _max_desired_young_size = MAX2(_min_desired_young_size, _max_desired_young_size); + break; + case SizerMaxNewSizeOnly: + _min_desired_young_size = calculate_min_size(heap_size); + _min_desired_young_size = MIN2(_min_desired_young_size, _max_desired_young_size); + break; + case SizerMaxAndNewSize: + // Do nothing. Values set on the command line, don't update them at runtime. + break; + case SizerNewRatio: + _min_desired_young_size = MAX2((heap_size / (NewRatio + 1)), ShenandoahHeapRegion::region_size_bytes()); + _max_desired_young_size = _min_desired_young_size; + break; + default: + ShouldNotReachHere(); + } + + assert(_min_desired_young_size <= _max_desired_young_size, "Invalid min/max young gen size values"); +} + +void ShenandoahGenerationSizer::heap_size_changed(size_t heap_size) { + recalculate_min_max_young_length(heap_size); +} + +bool ShenandoahGenerationSizer::adjust_generation_sizes() { shenandoah_assert_generational(); - if (_mmu_average.davg() >= double(GCTimeRatio)) { + if (!use_adaptive_sizing()) { + return false; + } + + if (_mmu_tracker->average() >= double(GCTimeRatio)) { return false; } ShenandoahHeap* heap = ShenandoahHeap::heap(); ShenandoahOldGeneration *old = heap->old_generation(); - double old_time_s = old->reset_collection_time(); ShenandoahYoungGeneration *young = heap->young_generation(); - double young_time_s = young->reset_collection_time(); ShenandoahGeneration *global = heap->global_generation(); + double old_time_s = old->reset_collection_time(); + double young_time_s = young->reset_collection_time(); double global_time_s = global->reset_collection_time(); + const double transfer_threshold = 3.0; + double delta = young_time_s - old_time_s; + log_info(gc)("Thread Usr+Sys YOUNG = %.3f, OLD = %.3f, GLOBAL = %.3f", young_time_s, old_time_s, global_time_s); - if (old_time_s > young_time_s) { - return transfer_capacity(young, old); - } else { - return transfer_capacity(old, young); + if (abs(delta) <= transfer_threshold) { + log_info(gc, ergo)("Difference (%.3f) for thread utilization for each generation is under threshold (%.3f)", abs(delta), transfer_threshold); + return false; } -} -size_t percentage_of_heap(size_t bytes) { - size_t heap_capacity = ShenandoahHeap::heap()->max_capacity(); - assert(bytes < heap_capacity, "Must be less than total capacity"); - return size_t(100.0 * double(bytes) / double(heap_capacity)); + if (delta > 0) { + // young is busier than old, increase size of young to raise MMU + return transfer_capacity(old, young); + } else { + // old is busier than young, increase size of old to raise MMU + return transfer_capacity(young, old); + } } -bool ShenandoahMmuTracker::transfer_capacity(ShenandoahGeneration* from, ShenandoahGeneration* to) { +bool ShenandoahGenerationSizer::transfer_capacity(ShenandoahGeneration* from, ShenandoahGeneration* to) { shenandoah_assert_heaplocked_or_safepoint(); size_t available_regions = from->free_unaffiliated_regions(); @@ -144,34 +245,16 @@ bool ShenandoahMmuTracker::transfer_capacity(ShenandoahGeneration* from, Shenand size_t regions_to_transfer = MAX2(1u, uint(double(available_regions) * _resize_increment)); size_t bytes_to_transfer = regions_to_transfer * ShenandoahHeapRegion::region_size_bytes(); if (from->generation_mode() == YOUNG) { - size_t new_young_size = from->max_capacity() - bytes_to_transfer; - if (percentage_of_heap(new_young_size) < ShenandoahMinYoungPercentage) { - ShenandoahHeap* heap = ShenandoahHeap::heap(); - size_t minimum_size = size_t(ShenandoahMinYoungPercentage / 100.0 * heap->max_capacity()); - if (from->max_capacity() > minimum_size) { - bytes_to_transfer = from->max_capacity() - minimum_size; - } else { - log_info(gc)("Cannot transfer from young: " SIZE_FORMAT "%s, at minimum capacity: " SIZE_FORMAT "%s", - byte_size_in_proper_unit(from->max_capacity()), proper_unit_for_byte_size(from->max_capacity()), - byte_size_in_proper_unit(minimum_size), proper_unit_for_byte_size(minimum_size)); - return false; - } - } + bytes_to_transfer = adjust_transfer_from_young(from, bytes_to_transfer); } else { - assert(to->generation_mode() == YOUNG, "Can only transfer between young and old."); - size_t new_young_size = to->max_capacity() + bytes_to_transfer; - if (percentage_of_heap(new_young_size) > ShenandoahMaxYoungPercentage) { - ShenandoahHeap* heap = ShenandoahHeap::heap(); - size_t maximum_size = size_t(ShenandoahMaxYoungPercentage / 100.0 * heap->max_capacity()); - if (maximum_size > to->max_capacity()) { - bytes_to_transfer = maximum_size - to->max_capacity(); - } else { - log_info(gc)("Cannot transfer to young: " SIZE_FORMAT "%s, at maximum capacity: " SIZE_FORMAT "%s", - byte_size_in_proper_unit(to->max_capacity()), proper_unit_for_byte_size(to->max_capacity()), - byte_size_in_proper_unit(maximum_size), proper_unit_for_byte_size(maximum_size)); - return false; - } - } + bytes_to_transfer = adjust_transfer_to_young(to, bytes_to_transfer); + } + + if (bytes_to_transfer == 0) { + log_debug(gc)("No capacity available to transfer from: %s (" SIZE_FORMAT "%s) to: %s (" SIZE_FORMAT "%s)", + from->name(), byte_size_in_proper_unit(from->max_capacity()), proper_unit_for_byte_size(from->max_capacity()), + to->name(), byte_size_in_proper_unit(to->max_capacity()), proper_unit_for_byte_size(to->max_capacity())); + return false; } assert(bytes_to_transfer <= regions_to_transfer * ShenandoahHeapRegion::region_size_bytes(), "Cannot transfer more than available in free regions."); @@ -182,9 +265,34 @@ bool ShenandoahMmuTracker::transfer_capacity(ShenandoahGeneration* from, Shenand return true; } -void ShenandoahMmuTracker::initialize() { - _initial_process_time_s = process_time_seconds(); - _initial_collector_time_s = gc_thread_time_seconds(); - _initial_verify_collector_time_s = _initial_collector_time_s; - _mmu_periodic_task->enroll(); -} \ No newline at end of file +size_t round_down_to_multiple_of_region_size(size_t bytes) { + return (bytes / ShenandoahHeapRegion::region_size_bytes()) * ShenandoahHeapRegion::region_size_bytes(); +} + +size_t ShenandoahGenerationSizer::adjust_transfer_from_young(ShenandoahGeneration* from, size_t bytes_to_transfer) const { + assert(from->generation_mode() == YOUNG, "Expect to transfer from young"); + size_t new_young_size = from->max_capacity() - bytes_to_transfer; + size_t minimum_size = min_young_size(); + // Check that we are not going to violate the minimum size constraint. + if (new_young_size < minimum_size) { + assert(minimum_size <= from->max_capacity(), "Young is under minimum capacity."); + // If the transfer violates the minimum size and there is still some capacity to transfer, + // adjust the transfer to take the size to the minimum. Note that this may be zero. + bytes_to_transfer = round_down_to_multiple_of_region_size(from->max_capacity() - minimum_size); + } + return bytes_to_transfer; +} + +size_t ShenandoahGenerationSizer::adjust_transfer_to_young(ShenandoahGeneration* to, size_t bytes_to_transfer) const { + assert(to->generation_mode() == YOUNG, "Can only transfer between young and old."); + size_t new_young_size = to->max_capacity() + bytes_to_transfer; + size_t maximum_size = max_young_size(); + // Check that we are not going to violate the maximum size constraint. + if (new_young_size > maximum_size) { + assert(maximum_size >= to->max_capacity(), "Young is over maximum capacity"); + // If the transfer violates the maximum size and there is still some capacity to transfer, + // adjust the transfer to take the size to the maximum. Note that this may be zero. + bytes_to_transfer = round_down_to_multiple_of_region_size(maximum_size - to->max_capacity()); + } + return bytes_to_transfer; +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.hpp index 1c9f20d9d2400..113d66442e22b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.hpp @@ -34,8 +34,10 @@ class ShenandoahMmuTask; /** * This class is responsible for tracking and adjusting the minimum mutator * utilization (MMU). MMU is defined as the percentage of CPU time available - * to mutator threads over an arbitrary, fixed interval of time. MMU is measured - * by summing all of the time given to the GC threads and comparing this too + * to mutator threads over an arbitrary, fixed interval of time. This interval + * defaults to 5 seconds and is configured by GCPauseIntervalMillis. The class + * maintains a decaying average of the last 10 values. The MMU is measured + * by summing all of the time given to the GC threads and comparing this to * the total CPU time for the process. There are OS APIs to support this on * all major platforms. * @@ -49,17 +51,13 @@ class ShenandoahMmuTask; */ class ShenandoahMmuTracker { - double _initial_collector_time_s; - double _initial_process_time_s; - double _initial_verify_collector_time_s; - - double _resize_increment; + double _generational_reference_time_s; + double _process_reference_time_s; + double _collector_reference_time_s; ShenandoahMmuTask* _mmu_periodic_task; TruncatedSeq _mmu_average; - bool transfer_capacity(ShenandoahGeneration* from, ShenandoahGeneration* to); - static double gc_thread_time_seconds(); static double process_time_seconds(); @@ -83,17 +81,75 @@ class ShenandoahMmuTracker { // This method also logs the average MMU. void report(); + double average() { + return _mmu_average.davg(); + } +}; + +class ShenandoahGenerationSizer { +private: + enum SizerKind { + SizerDefaults, + SizerNewSizeOnly, + SizerMaxNewSizeOnly, + SizerMaxAndNewSize, + SizerNewRatio + }; + SizerKind _sizer_kind; + + // False when using a fixed young generation size due to command-line options, + // true otherwise. + bool _use_adaptive_sizing; + + size_t _min_desired_young_size; + size_t _max_desired_young_size; + + double _resize_increment; + ShenandoahMmuTracker* _mmu_tracker; + + static size_t calculate_min_size(size_t heap_size); + static size_t calculate_max_size(size_t heap_size); + + // Update the given values for minimum and maximum young gen length in regions + // given the number of heap regions depending on the kind of sizing algorithm. + void recalculate_min_max_young_length(size_t heap_size); + + // This will attempt to transfer capacity from one generation to the other. It + // returns true if a transfer is made, false otherwise. + bool transfer_capacity(ShenandoahGeneration* from, ShenandoahGeneration* to); + + // These two methods are responsible for enforcing the minimum and maximum + // constraints for the size of the generations. + size_t adjust_transfer_from_young(ShenandoahGeneration* from, size_t bytes_to_transfer) const; + size_t adjust_transfer_to_young(ShenandoahGeneration* to, size_t bytes_to_transfer) const; + +public: + ShenandoahGenerationSizer(ShenandoahMmuTracker* mmu_tracker); + + // Calculate the maximum length of the young gen given the number of regions + // depending on the sizing algorithm. + void heap_size_changed(size_t heap_size); + + size_t min_young_size() const { + return _min_desired_young_size; + } + size_t max_young_size() const { + return _max_desired_young_size; + } + + bool use_adaptive_sizing() const { + return _use_adaptive_sizing; + } + // This is invoked at the end of a collection. This happens on a safepoint // to avoid any races with allocators (and to avoid interfering with // allocators by taking the heap lock). The amount of capacity to move // from one generation to another is controlled by YoungGenerationSizeIncrement - // and defaults to 20% of the heap. The minimum and maximum sizes of the - // young generation are controlled by ShenandoahMinYoungPercentage and - // ShenandoahMaxYoungPercentage, respectively. The method returns true - // when and adjustment is made, false otherwise. + // and defaults to 20% of the available capacity of the donor generation. + // The minimum and maximum sizes of the young generation are controlled by + // ShenandoahMinYoungPercentage and ShenandoahMaxYoungPercentage, respectively. + // The method returns true when an adjustment is made, false otherwise. bool adjust_generation_sizes(); }; - - #endif //SHARE_GC_SHENANDOAH_SHENANDOAHMMUTRACKER_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp index f7f70bfdf1b6b..dfa12f4239a23 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp @@ -44,7 +44,7 @@ const char* ShenandoahYoungGeneration::name() const { void ShenandoahYoungGeneration::set_concurrent_mark_in_progress(bool in_progress) { ShenandoahHeap* heap = ShenandoahHeap::heap(); heap->set_concurrent_young_mark_in_progress(in_progress); - if (_old_gen_task_queues != nullptr && in_progress && !heap->is_prepare_for_old_mark_in_progress()) { + if (is_bootstrap_cycle() && in_progress && !heap->is_prepare_for_old_mark_in_progress()) { // This is not a bug. When the bootstrapping marking phase is complete, // the old generation marking is still in progress, unless it's not. // In the case that old-gen preparation for mixed evacuation has been @@ -75,7 +75,7 @@ bool ShenandoahYoungGeneration::is_concurrent_mark_in_progress() { void ShenandoahYoungGeneration::reserve_task_queues(uint workers) { ShenandoahGeneration::reserve_task_queues(workers); - if (_old_gen_task_queues != NULL) { + if (is_bootstrap_cycle()) { _old_gen_task_queues->reserve(workers); } } @@ -92,7 +92,7 @@ ShenandoahHeuristics* ShenandoahYoungGeneration::initialize_heuristics(Shenandoa } void ShenandoahYoungGeneration::add_collection_time(double time_seconds) { - if (_old_gen_task_queues != NULL) { + if (is_bootstrap_cycle()) { // This is a bootstrap cycle, so attribute time to old gc ShenandoahHeap::heap()->old_generation()->add_collection_time(time_seconds); } else { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp index bf517647aa3ca..191a3f9046d10 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp @@ -52,6 +52,12 @@ class ShenandoahYoungGeneration : public ShenandoahGeneration { return _old_gen_task_queues; } + // Returns true if the young generation is configured to enqueue old + // oops for the old generation mark queues. + bool is_bootstrap_cycle() { + return _old_gen_task_queues != nullptr; + } + void reserve_task_queues(uint workers) override; virtual ShenandoahHeuristics* initialize_heuristics(ShenandoahMode* gc_mode) override; diff --git a/test/hotspot/gtest/gc/shenandoah/test_shenandoahOldHeuristic.cpp b/test/hotspot/gtest/gc/shenandoah/test_shenandoahOldHeuristic.cpp index 8ac2926bb5f78..eca43480b9f9a 100644 --- a/test/hotspot/gtest/gc/shenandoah/test_shenandoahOldHeuristic.cpp +++ b/test/hotspot/gtest/gc/shenandoah/test_shenandoahOldHeuristic.cpp @@ -281,3 +281,5 @@ TEST_VM_F(ShenandoahOldHeuristicTest, unpinned_region_is_middle) { EXPECT_EQ(_collection_set->get_old_garbage(), g1 + g3); EXPECT_EQ(_heuristics->unprocessed_old_collection_candidates(), 0UL); } + +#undef SKIP_IF_NOT_SHENANDOAH \ No newline at end of file From d793fd1620e430be6fcba83361b421856f5bbfd5 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Wed, 21 Dec 2022 23:41:23 +0000 Subject: [PATCH 166/254] Avoid divide by zero error, improve variable names Reviewed-by: kdnilsen --- .../gc/shenandoah/shenandoahMmuTracker.cpp | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp index d981efba0bde7..34835db1435b6 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp @@ -28,9 +28,9 @@ #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahOldGeneration.hpp" #include "gc/shenandoah/shenandoahYoungGeneration.hpp" +#include "logging/log.hpp" #include "runtime/os.hpp" #include "runtime/task.hpp" -#include "logging/log.hpp" class ShenandoahMmuTask : public PeriodicTask { @@ -94,12 +94,17 @@ void ShenandoahMmuTracker::report() { // This is only called by the periodic thread. double process_time_s = process_time_seconds(); double elapsed_process_time_s = process_time_s - _process_reference_time_s; + if (elapsed_process_time_s <= 0.01) { + // No cpu time for this interval? + return; + } + _process_reference_time_s = process_time_s; - double verify_time_s = gc_thread_time_seconds(); - double verify_elapsed = verify_time_s - _collector_reference_time_s; - _collector_reference_time_s = verify_time_s; - double verify_mmu = ((elapsed_process_time_s - verify_elapsed) / elapsed_process_time_s) * 100; - _mmu_average.add(verify_mmu); + double collector_time_s = gc_thread_time_seconds(); + double elapsed_collector_time_s = collector_time_s - _collector_reference_time_s; + _collector_reference_time_s = collector_time_s; + double minimum_mutator_utilization = ((elapsed_process_time_s - elapsed_collector_time_s) / elapsed_process_time_s) * 100; + _mmu_average.add(minimum_mutator_utilization); log_info(gc)("Average MMU = %.3f", _mmu_average.davg()); } From 6c8fa0f735bbc9f80f628145867db8bed5d074f4 Mon Sep 17 00:00:00 2001 From: "Y. Srinivas Ramakrishna" Date: Wed, 28 Dec 2022 00:41:20 +0000 Subject: [PATCH 167/254] 8297796: GenShen: instrument the remembered set scan Reviewed-by: wkemper, kdnilsen --- .../gc/shenandoah/shenandoahCardStats.cpp | 126 ++++++ .../gc/shenandoah/shenandoahCardStats.hpp | 170 ++++++++ .../gc/shenandoah/shenandoahGeneration.cpp | 4 + .../share/gc/shenandoah/shenandoahHeap.cpp | 21 +- .../shenandoah/shenandoahScanRemembered.cpp | 26 +- .../shenandoah/shenandoahScanRemembered.hpp | 402 ++++++++---------- .../shenandoahScanRemembered.inline.hpp | 264 +++++++----- .../gc/shenandoah/shenandoah_globals.hpp | 11 +- src/hotspot/share/utilities/numberSeq.cpp | 4 +- 9 files changed, 658 insertions(+), 370 deletions(-) create mode 100644 src/hotspot/share/gc/shenandoah/shenandoahCardStats.cpp create mode 100644 src/hotspot/share/gc/shenandoah/shenandoahCardStats.hpp diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCardStats.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCardStats.cpp new file mode 100644 index 0000000000000..e87c1ae75cee1 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahCardStats.cpp @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2022, Amazon.com, Inc. or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + + +#include "precompiled.hpp" + +#include "gc/shenandoah/shenandoahCardStats.hpp" +#include "logging/log.hpp" + +#ifndef PRODUCT +void ShenandoahCardStats::update_run_work(bool record) { + assert(!(_last_dirty || _last_clean) || (_last_dirty && _dirty_run > 0) || (_last_clean && _clean_run > 0), + "dirty/clean run stats inconsistent"); + assert(_dirty_run == 0 || _clean_run == 0, "Both shouldn't be non-zero"); + if (_dirty_run > _max_dirty_run) { + assert(_last_dirty, "Error"); + _max_dirty_run = _dirty_run; + } else if (_clean_run > _max_clean_run) { + assert(_last_clean, "Error"); + _max_clean_run = _clean_run; + } + _dirty_card_cnt += _dirty_run; + _clean_card_cnt += _clean_run; + + // Update local stats + { + assert(_dirty_run <= _cards_in_cluster, "Error"); + assert(_clean_run <= _cards_in_cluster, "Error"); + // Update global stats for distribution of dirty/clean run lengths + _local_card_stats[DIRTY_RUN].add((double)_dirty_run*100/(double)_cards_in_cluster); + _local_card_stats[CLEAN_RUN].add((double)_clean_run*100/(double)_cards_in_cluster); + + if (record) { + // Update global stats for distribution of dirty/clean cards as a percentage of chunk + _local_card_stats[DIRTY_CARDS].add((double)_dirty_card_cnt*100/(double)_cards_in_cluster); + _local_card_stats[CLEAN_CARDS].add((double)_clean_card_cnt*100/(double)_cards_in_cluster); + + // Update global stats for max dirty/clean run distribution as a percentage of chunk + _local_card_stats[MAX_DIRTY_RUN].add((double)_max_dirty_run*100/(double)_cards_in_cluster); + _local_card_stats[MAX_CLEAN_RUN].add((double)_max_clean_run*100/(double)_cards_in_cluster); + + // Update global stats for dirty & clean object counts + _local_card_stats[DIRTY_OBJS].add(_dirty_obj_cnt); + _local_card_stats[CLEAN_OBJS].add(_clean_obj_cnt); + _local_card_stats[DIRTY_SCANS].add(_dirty_scan_cnt); + _local_card_stats[CLEAN_SCANS].add(_clean_scan_cnt); + + _local_card_stats[ALTERNATIONS].add(_alternation_cnt); + } + } + + if (record) { + // reset the stats for the next cluster + _dirty_card_cnt = 0; + _clean_card_cnt = 0; + + _max_dirty_run = 0; + _max_clean_run = 0; + + _dirty_obj_cnt = 0; + _clean_obj_cnt = 0; + + _dirty_scan_cnt = 0; + _clean_scan_cnt = 0; + + _alternation_cnt = 0; + } + _dirty_run = 0; + _clean_run = 0; + _last_dirty = false; + _last_clean = false; + assert(!record || is_clean(), "Error"); +} + +bool ShenandoahCardStats::is_clean() { + return + _dirty_card_cnt == 0 && + _clean_card_cnt == 0 && + _max_dirty_run == 0 && + _max_clean_run == 0 && + _dirty_obj_cnt == 0 && + _clean_obj_cnt == 0 && + _dirty_scan_cnt == 0 && + _clean_scan_cnt == 0 && + _alternation_cnt == 0 && + _dirty_run == 0 && + _clean_run == 0 && + _last_dirty == false && + _last_clean == false; +} + +void ShenandoahCardStats::log() const { + if (ShenandoahEnableCardStats) { + log_info(gc,remset)("Card stats: dirty " SIZE_FORMAT " (max run: " SIZE_FORMAT ")," + " clean " SIZE_FORMAT " (max run: " SIZE_FORMAT ")," + " dirty objs " SIZE_FORMAT ", clean objs " SIZE_FORMAT "," + " dirty scans " SIZE_FORMAT ", clean scans " SIZE_FORMAT, + _dirty_card_cnt, _max_dirty_run, _clean_card_cnt, _max_clean_run, + _dirty_obj_cnt, _clean_obj_cnt, + _dirty_scan_cnt, _clean_scan_cnt); + } +} +#endif // !PRODUCT + diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCardStats.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCardStats.hpp new file mode 100644 index 0000000000000..146348b78348b --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahCardStats.hpp @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2022, Amazon.com, Inc. or its affiliates. All rights reserved. + * + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHCARDSTATS_HPP +#define SHARE_GC_SHENANDOAH_SHENANDOAHCARDSTATS_HPP + +#include "gc/shared/gc_globals.hpp" +#include "gc/shenandoah/shenandoahNumberSeq.hpp" + +class ShenandoahCardStats: public CHeapObj { +private: + size_t _cards_in_cluster; + HdrSeq* _local_card_stats; + + bool _last_dirty; + bool _last_clean; + + size_t _dirty_card_cnt; + size_t _clean_card_cnt; + + size_t _dirty_run; + size_t _clean_run; + + size_t _max_dirty_run; + size_t _max_clean_run; + + size_t _dirty_obj_cnt; + size_t _clean_obj_cnt; + + size_t _dirty_scan_cnt; + size_t _clean_scan_cnt; + + size_t _alternation_cnt; + +public: + ShenandoahCardStats(size_t cards_in_cluster, HdrSeq* card_stats) : + _cards_in_cluster(cards_in_cluster), + _local_card_stats(card_stats), + _last_dirty(false), + _last_clean(false), + _dirty_card_cnt(0), + _clean_card_cnt(0), + _dirty_run(0), + _clean_run(0), + _max_dirty_run(0), + _max_clean_run(0), + _dirty_obj_cnt(0), + _clean_obj_cnt(0), + _dirty_scan_cnt(0), + _clean_scan_cnt(0), + _alternation_cnt(0) + { } + +private: + void increment_card_cnt_work(bool dirty) { + if (dirty) { // dirty card + if (_last_dirty) { + assert(_dirty_run > 0 && _clean_run == 0 && !_last_clean, "Error"); + _dirty_run++; + } else { + if (_last_clean) { + _alternation_cnt++; + } + update_run(false); + _last_dirty = true; + _dirty_run = 1; + } + } else { // clean card + if (_last_clean) { + assert(_clean_run > 0 && _dirty_run == 0 && !_last_dirty, "Error"); + _clean_run++; + } else { + if (_last_dirty) { + _alternation_cnt++; + } + update_run(false); + _last_clean = true; + _clean_run = 1; + } + } + } + + inline void increment_obj_cnt_work(bool dirty) { + assert(!dirty || (_last_dirty && _dirty_run > 0), "Error"); + assert(dirty || (_last_clean && _clean_run > 0), "Error"); + dirty ? _dirty_obj_cnt++ : _clean_obj_cnt++; + } + + inline void increment_scan_cnt_work(bool dirty) { + assert(!dirty || (_last_dirty && _dirty_run > 0), "Error"); + assert(dirty || (_last_clean && _clean_run > 0), "Error"); + dirty ? _dirty_scan_cnt++ : _clean_scan_cnt++; + } + + void update_run_work(bool cluster) PRODUCT_RETURN; + +public: + inline void increment_card_cnt(bool dirty) { + if (ShenandoahEnableCardStats) { + increment_card_cnt_work(dirty); + } + } + + inline void increment_obj_cnt(bool dirty) { + if (ShenandoahEnableCardStats) { + increment_obj_cnt_work(dirty); + } + } + + inline void increment_scan_cnt(bool dirty) { + if (ShenandoahEnableCardStats) { + increment_scan_cnt_work(dirty); + } + } + + inline void update_run(bool record) { + if (ShenandoahEnableCardStats) { + update_run_work(record); + } + } + + bool is_clean() PRODUCT_RETURN0; + + void log() const PRODUCT_RETURN; +}; + +enum CardStatType { + DIRTY_RUN = 0, + CLEAN_RUN = 1, + DIRTY_CARDS = 2, + CLEAN_CARDS = 3, + MAX_DIRTY_RUN = 4, + MAX_CLEAN_RUN = 5, + DIRTY_OBJS = 6, + CLEAN_OBJS = 7, + DIRTY_SCANS = 8, + CLEAN_SCANS= 9, + ALTERNATIONS = 10, + MAX_CARD_STAT_TYPE = 11 +}; + +enum CardStatLogType { + CARD_STAT_SCAN_RS = 0, + CARD_STAT_UPDATE_REFS = 1, + MAX_CARD_STAT_LOG_TYPE = 2 +}; + +#endif // SHARE_GC_SHENANDOAH_SHENANDOAHCARDSTATS_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index cb3b2c0a7358c..67ed1f0e4115d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -912,6 +912,10 @@ void ShenandoahGeneration::scan_remembered_set(bool is_concurrent) { ShenandoahScanRememberedTask task(task_queues(), old_gen_task_queues(), rp, &work_list, is_concurrent); heap->assert_gc_workers(nworkers); heap->workers()->run_task(&task); + if (ShenandoahEnableCardStats) { + assert(heap->card_scan() != NULL, "Not generational"); + heap->card_scan()->log_card_stats(nworkers, CARD_STAT_SCAN_RS); + } } size_t ShenandoahGeneration::increment_affiliated_region_count() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index e1841d986c2dd..119588089363f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -2674,6 +2674,7 @@ class ShenandoahUpdateHeapRefsTask : public WorkerTask { } r = _regions->next(); } + if (_heap->mode()->is_generational() && (_heap->active_generation()->generation_mode() != GLOBAL)) { // Since this is generational and not GLOBAL, we have to process the remembered set. There's no remembered // set processing if not in generational mode or if GLOBAL mode. @@ -2682,9 +2683,10 @@ class ShenandoahUpdateHeapRefsTask : public WorkerTask { // The remembered set workload is better balanced between threads, so threads that are "behind" can catch up with other // threads during this phase, allowing all threads to work more effectively in parallel. struct ShenandoahRegionChunk assignment; - bool have_work = _work_chunks->next(&assignment); RememberedScanner* scanner = _heap->card_scan(); - while (have_work) { + + while (!_heap->check_cancelled_gc_and_yield(CONCURRENT) && _work_chunks->next(&assignment)) { + // Keep grabbing next work chunk to process until finished, or asked to yield ShenandoahHeapRegion* r = assignment._r; if (r->is_active() && !r->is_cset() && (r->affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION)) { HeapWord* start_of_range = r->bottom() + assignment._chunk_offset; @@ -2777,20 +2779,13 @@ class ShenandoahUpdateHeapRefsTask : public WorkerTask { CardTable::card_size_in_words() * ShenandoahCardCluster::CardsPerCluster; size_t clusters = assignment._chunk_size / cluster_size; assert(clusters * cluster_size == assignment._chunk_size, "Chunk assignment must align on cluster boundaries"); - scanner->process_region_slice(r, assignment._chunk_offset, clusters, end_of_range, &cl, true, CONCURRENT); + scanner->process_region_slice(r, assignment._chunk_offset, clusters, end_of_range, &cl, true, CONCURRENT, worker_id); } } if (ShenandoahPacing && (start_of_range < end_of_range)) { _heap->pacer()->report_updaterefs(pointer_delta(end_of_range, start_of_range)); } } - // Otherwise, this work chunk had nothing for me to do, so do not report pacer progress. - - // Before we take responsibility for another chunk of work, see if cancellation is requested. - if (_heap->check_cancelled_gc_and_yield(CONCURRENT)) { - return; - } - have_work = _work_chunks->next(&assignment); } } } @@ -2798,7 +2793,8 @@ class ShenandoahUpdateHeapRefsTask : public WorkerTask { void ShenandoahHeap::update_heap_references(bool concurrent) { assert(!is_full_gc_in_progress(), "Only for concurrent and degenerated GC"); - ShenandoahRegionChunkIterator work_list(workers()->active_workers()); + uint nworkers = workers()->active_workers(); + ShenandoahRegionChunkIterator work_list(nworkers); if (concurrent) { ShenandoahUpdateHeapRefsTask task(&_update_refs_iterator, &work_list); @@ -2807,6 +2803,9 @@ void ShenandoahHeap::update_heap_references(bool concurrent) { ShenandoahUpdateHeapRefsTask task(&_update_refs_iterator, &work_list); workers()->run_task(&task); } + if (ShenandoahEnableCardStats && card_scan()!=NULL) { // generational check proxy + card_scan()->log_card_stats(nworkers, CARD_STAT_UPDATE_REFS); + } } class ShenandoahFinalUpdateRefsUpdateRegionStateClosure : public ShenandoahHeapRegionClosure { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp index f05e9988a4bb1..6d6441cbaf3cf 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp @@ -30,6 +30,7 @@ #include "gc/shenandoah/shenandoahOopClosures.inline.hpp" #include "gc/shenandoah/shenandoahReferenceProcessor.hpp" #include "gc/shenandoah/shenandoahScanRemembered.inline.hpp" +#include "logging/log.hpp" ShenandoahDirectCardMarkRememberedSet::ShenandoahDirectCardMarkRememberedSet(ShenandoahCardTable* card_table, size_t total_card_count) { _heap = ShenandoahHeap::heap(); @@ -79,15 +80,7 @@ void ShenandoahScanRememberedTask::do_work(uint worker_id) { // set up thread local closure for shen ref processor _rp->set_mark_closure(worker_id, &cl); struct ShenandoahRegionChunk assignment; - bool has_work = _work_list->next(&assignment); - while (has_work) { -#ifdef ENABLE_REMEMBERED_SET_CANCELLATION - // This check is currently disabled to avoid crashes that occur - // when we try to cancel remembered set scanning - if (heap->check_cancelled_gc_and_yield(_is_concurrent)) { - return; - } -#endif + while (_work_list->next(&assignment)) { ShenandoahHeapRegion* region = assignment._r; log_debug(gc)("ShenandoahScanRememberedTask::do_work(%u), processing slice of region " SIZE_FORMAT " at offset " SIZE_FORMAT ", size: " SIZE_FORMAT, @@ -99,13 +92,22 @@ void ShenandoahScanRememberedTask::do_work(uint worker_id) { assert(clusters * cluster_size == assignment._chunk_size, "Chunk assignments must align on cluster boundaries"); HeapWord* end_of_range = region->bottom() + assignment._chunk_offset + assignment._chunk_size; - // During concurrent mark, region->top() equals TAMS with respect to the current young-gen pass. */ + // During concurrent mark, region->top() equals TAMS with respect to the current young-gen pass. if (end_of_range > region->top()) { end_of_range = region->top(); } - scanner->process_region_slice(region, assignment._chunk_offset, clusters, end_of_range, &cl, false, _is_concurrent); + scanner->process_region_slice(region, assignment._chunk_offset, clusters, end_of_range, &cl, false, _is_concurrent, worker_id); } - has_work = _work_list->next(&assignment); +#ifdef ENABLE_REMEMBERED_SET_CANCELLATION + // This check is currently disabled to avoid crashes that occur + // when we try to cancel remembered set scanning; it should be re-enabled + // after the issues are fixed, as it would allow more prompt cancellation and + // transition to degenerated / full GCs. Note that work that has been assigned/ + // claimed above must be completed before we return here upon cancellation. + if (heap->check_cancelled_gc_and_yield(_is_concurrent)) { + return; + } +#endif } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp index d42e9913cb6f9..b372d1cc17910 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp @@ -45,17 +45,23 @@ // many references are held within the objects that span a // DIRTY card's memory, and on the size of the object // that spans the end of a DIRTY card's memory (because -// that object will be scanned in its entirety). For these -// reasons, it is advisable for the multiple worker threads -// to be flexible in the number of clusters to be -// processed by each thread. +// that object, if it's not an array, may need to be scanned in +// its entirety, when the object is imprecisely dirtied. Imprecise +// dirtying is when the card corresponding to the object header +// is dirtied, rather than the card on which the updated field lives). +// To better balance work amongst them, parallel worker threads dynamically +// claim clusters and are flexible in the number of clusters they +// process. // // A cluster represents a "natural" quantum of work to be performed by // a parallel GC thread's background remembered set scanning efforts. // The notion of cluster is similar to the notion of stripe in the // implementation of parallel GC card scanning. However, a cluster is // typically smaller than a stripe, enabling finer grain division of -// labor between multiple threads. +// labor between multiple threads, and potentially better load balancing +// when dirty cards are not uniformly distributed in the heap, as is often +// the case with generational workloads where more recently promoted objects +// may be dirtied more frequently that older objects. // // For illustration, consider the following possible JVM configurations: // @@ -63,28 +69,28 @@ // RegionSize is 128 MB // Span of a card entry is 512 B // Each card table entry consumes 1 B -// Assume one long word of card table entries represents a cluster. +// Assume one long word (8 B)of the card table represents a cluster. // This long word holds 8 card table entries, spanning a -// total of 4KB -// The number of clusters per region is 128 MB / 4 KB = 32K +// total of 8*512 B = 4 KB of the heap +// The number of clusters per region is 128 MB / 4 KB = 32 K // // Scenario 2: // RegionSize is 128 MB // Span of each card entry is 128 B // Each card table entry consumes 1 bit -// Assume one int word of card tables represents a cluster. -// This int word holds 32 card table entries, spanning a -// total of 4KB -// The number of clusters per region is 128 MB / 4 KB = 32K +// Assume one int word (4 B) of the card table represents a cluster. +// This int word holds 32 b/1 b = 32 card table entries, spanning a +// total of 32 * 128 B = 4 KB of the heap +// The number of clusters per region is 128 MB / 4 KB = 32 K // // Scenario 3: // RegionSize is 128 MB // Span of each card entry is 512 B // Each card table entry consumes 1 bit -// Assume one long word of card tables represents a cluster. -// This long word holds 64 card table entries, spanning a -// total of 32 KB -// The number of clusters per region is 128 MB / 32 KB = 4K +// Assume one long word (8 B) of card table represents a cluster. +// This long word holds 64 b/ 1 b = 64 card table entries, spanning a +// total of 64 * 512 B = 32 KB of the heap +// The number of clusters per region is 128 MB / 32 KB = 4 K // // At the start of a new young-gen concurrent mark pass, the gang of // Shenandoah worker threads collaborate in performing the following @@ -103,21 +109,6 @@ // determine the cluster number of the first cluster belonging // to that region // for each cluster contained within that region -// Assure that exactly one worker thread initializes each -// cluster of overreach memory by invoking: -// -// rs->initialize_overreach(cluster_no, cluster_count) -// -// in separate threads. (Divide up the clusters so that -// different threads are responsible for initializing different -// clusters. Initialization cost is essentially identical for -// each cluster.) -// -// Next, we repeat the process for invocations of process_clusters. -// for each ShenandoahHeapRegion old_region in the whole heap -// determine the cluster number of the first cluster belonging -// to that region -// for each cluster contained within that region // Assure that exactly one worker thread processes each // cluster, each thread making a series of invocations of the // following: @@ -134,18 +125,17 @@ // clusters contain mostly clean cards // b) some clusters contain mostly primitive data and other // clusters contain mostly reference data -// c) some clusters are spanned by very large objects that -// begin in some other cluster. When a large object +// c) some clusters are spanned by very large non-array objects that +// begin in some other cluster. When a large non-array object // beginning in a preceding cluster spans large portions of -// this cluster, the processing of this cluster gets a -// "free ride" because the thread responsible for processing -// the cluster that holds the object's header does the -// processing. +// this cluster, then because of imprecise dirtying, the +// portion of the object in this cluster may be clean, but +// will need to be processed by the worker responsible for +// this cluster, potentially increasing its work. // d) in the case that the end of this cluster is spanned by a -// very large object, the processing of this cluster will -// be responsible for examining the entire object, -// potentially requiring this thread to process large amounts -// of memory pertaining to other clusters. +// very large non-array object, the worker for this cluster will +// be responsible for processing the portion of the object +// in this cluster. // // Though an initial division of labor between marking threads may // assign equal numbers of clusters to be scanned by each thread, it @@ -177,39 +167,21 @@ // on to work on other tasks associated with root scanning until such // time as all clusters have been examined. // -// Once all clusters have been processed, the gang of GC worker -// threads collaborate to merge the overreach data. -// -// for each ShenandoahHeapRegion old_region in the whole heap -// determine the cluster number of the first cluster belonging -// to that region -// for each cluster contained within that region -// Assure that exactly one worker thread initializes each -// cluster of overreach memory by invoking: -// -// rs->merge_overreach(cluster_no, cluster_count) -// -// in separate threads. (Divide up the clusters so that -// different threads are responsible for merging different -// clusters. Merging cost is essentially identical for -// each cluster.) -// -// Though remembered set scanning is designed to run concurrently with -// mutator threads, the current implementation of remembered set -// scanning runs in parallel during a GC safepoint. Furthermore, the +// Remembered set scanning is designed to run concurrently with +// mutator threads, with multiple concurrent workers. Furthermore, the // current implementation of remembered set scanning never clears a -// card once it has been marked. Since the current implementation -// never clears marked pages, the current implementation does not -// invoke initialize_overreach() or merge_overreach(). +// card once it has been marked. // // These limitations will be addressed in future enhancements to the // existing implementation. #include #include "gc/shared/workerThread.hpp" +#include "gc/shenandoah/shenandoahCardStats.hpp" #include "gc/shenandoah/shenandoahCardTable.hpp" #include "gc/shenandoah/shenandoahHeap.hpp" #include "gc/shenandoah/shenandoahHeapRegion.hpp" +#include "gc/shenandoah/shenandoahNumberSeq.hpp" #include "gc/shenandoah/shenandoahTaskqueue.hpp" #include "memory/iterator.hpp" @@ -320,15 +292,7 @@ class ShenandoahDirectCardMarkRememberedSet: public CHeapObj { // The API services represent a compromise between efficiency and // convenience. // -// In the initial implementation, we assume that scanning of card -// table entries occurs only while the JVM is at a safe point. Thus, -// there is no synchronization required between GC threads that are -// scanning card-table entries and marking certain entries that were -// previously dirty as clean, and mutator threads which would possibly -// be marking certain card-table entries as dirty. -// -// There is however a need to implement concurrency control and memory -// coherency between multiple GC threads that scan the remembered set +// Multiple GC threads that scan the remembered set // in parallel. The desire is to divide the complete scanning effort // into multiple clusters of work that can be independently processed // by individual threads without need for synchronizing efforts @@ -337,93 +301,29 @@ class ShenandoahDirectCardMarkRememberedSet: public CHeapObj { // of Parallel GC. // // Complexity arises when an object to be scanned crosses the boundary -// between adjacent cluster regions. Here is the protocol that is -// followed: +// between adjacent cluster regions. Here is the protocol that we currently +// follow: // -// 1. We implement a supplemental data structure known as the overreach -// card table. The thread that is responsible for scanning each -// cluster of card-table entries is granted exclusive access to -// modify the associated card-table entries. In the case that a -// thread scans a very large object that reaches into one or more -// following clusters, that thread has exclusive access to the -// overreach card table for all of the entries belonging to the -// following clusters that are spanned by this large object. -// After all clusters have been scanned, the scanning threads -// briefly synchronize to merge the contents of the overreach -// entries with the traditional card table entries using logical- -// and operations. -// 2. Every object is scanned in its "entirety" by the thread that is -// responsible for the cluster that holds its starting address. -// Entirety is in quotes because there are various situations in -// which some portions of the object will not be scanned by this -// thread: -// a) If an object spans multiple card regions, all of which are -// contained within the same cluster, the scanning thread -// consults the existing card-table entries and does not scan -// portions of the object that are not currently dirty. -// b) For any cluster that is spanned in its entirety by a very -// large object, the GC thread that scans this object assumes -// full responsibility for maintenance of the associated -// card-table entries. -// c) If a cluster is partially spanned by an object originating -// in a preceding cluster, the portion of the object that -// partially spans the following cluster is scanned in its -// entirety (because the thread that is responsible for -// scanning the object cannot rely upon the card-table entries -// associated with the following cluster). Whenever references -// to young-gen memory are found within the scanned data, the -// associated overreach card table entries are marked as dirty -// by the scanning thread. -// 3. If a cluster is spanned in its entirety by an object that -// originates within a preceding cluster's memory, the thread -// assigned to examine this cluster does absolutely nothing. The -// thread assigned to scan the cluster that holds the object's -// starting address takes full responsibility for scanning the -// entire object and updating the associated card-table entries. -// 4. If a cluster is spanned partially by an object that originates -// within a preceding cluster's memory, the thread assigned to -// examine this cluster marks the card-table entry as clean for -// each card table that is fully spanned by this overreaching -// object. If a card-table entry's memory is partially spanned -// by the overreaching object, the thread sets the card-table -// entry to clean if it was previously dirty and if the portion -// of the card-table entry's memory that is not spanned by the -// overreaching object does not hold pointers to young-gen -// memory. -// 5. While examining a particular card belonging to a particular -// cluster, if an object reaches beyond the end of its card -// memory, the thread "scans" all portions of the object that -// correspond to DIRTY card entries within the current cluster and -// all portions of the object that reach into following clustesr. -// After this object is scanned, continue scanning with the memory -// that follows this object if this memory pertains to the same -// cluster. Otherwise, consider this cluster's memory to have -// been fully examined. +// 1. The thread responsible for scanning the cards in a cluster modifies +// the associated card-table entries. Only cards that are dirty are +// processed, except as described below for the case of objects that +// straddle more than one card. +// 2. Object Arrays are precisely dirtied, so only the portion of the obj-array +// that overlaps the range of dirty cards in its cluster are scanned +// by each worker thread. This holds for portions of obj-arrays that extend +// over clusters processed by different workers, with each worked responsible +// for scanning the portion of the obj-array overlapping the dirty cards in +// its cluster. +// 3. Non-array objects are precisely dirtied by the interpreter and the compilers +// For such objects that extend over multiple cards, or even multiple clusters, +// the entire object is scanned by the worker that processes the (dirty) card on +// which the object's header lies. (However, GC workers should precisely dirty the +// cards with inter-regional/inter-generational pointers in the body of this object, +// thus making subsequent scans potentially less expensive.) Such larger non-array +// objects are relatively rare. // -// Discussion: -// Though this design results from careful consideration of multiple -// design objectives, it is subject to various criticisms. Some -// discussion of the design choices is provided here: -// -// 1. Note that remembered sets are a heuristic technique to avoid -// the need to scan all of old-gen memory with each young-gen -// collection. If we sometimes scan a bit more memory than is -// absolutely necessary, that should be considered a reasonable -// compromise. This compromise is already present in the sizing -// of card table memory areas. Note that a single dirty pointer -// within a 512-byte card region forces the "unnecessary" scanning -// of 63 = ((512 - 8 = 504) / 8) pointers. -// 2. One undesirable aspect of this design is that we sometimes have -// to scan large amounts of memory belonging to very large -// objects, even for parts of the very large object that do not -// correspond to dirty card table entries. Note that this design -// limits the amount of non-dirty scanning that might have to -// be performed for these very large objects. In particular, only -// the last part of the very large object that extends into but -// does not completely span a particular cluster is unnecessarily -// scanned. Thus, for each very large object, the maximum -// over-scan is the size of memory spanned by a single cluster. -// 3. The representation of pointer location descriptive information +// A possible criticism: +// C. The representation of pointer location descriptive information // within Klass representations is not designed for efficient // "random access". An alternative approach to this design would // be to scan very large objects multiple times, once for each @@ -453,36 +353,22 @@ class ShenandoahDirectCardMarkRememberedSet: public CHeapObj { // represent old-gen memory, the card regions that span the // boundary between these neighboring heap regions will be // consecutively numbered. -// 3. (A corollary) In the case that an old-gen object spans the +// 3. (A corollary) In the case that an old-gen object straddles the // boundary between two heap regions, the card regions that // correspond to the span of this object will be consecutively // numbered. - - +// // ShenandoahCardCluster abstracts access to the remembered set // and also keeps track of crossing map information to allow efficient // resolution of object start addresses. // // ShenandoahCardCluster supports all of the services of // RememberedSet, plus it supports register_object() and lookup_object(). -// -// There are two situations under which we need to know the location -// at which the object spanning the start of a particular card-table -// memory region begins: -// -// 1. When we begin to scan dirty card memory that is not the -// first card region within a cluster, and the object that -// crosses into this card memory was not previously scanned, -// we need to find where that object starts so we can scan it. -// (Asides: if the objects starts within a previous cluster, it -// has already been scanned. If the object starts within this -// cluster and it spans at least one card region that is dirty -// and precedes this card region within the cluster, then it has -// already been scanned.) -// 2. When we are otherwise done scanning a complete cluster, if the -// last object within the cluster reaches into the following -// cluster, we need to scan this object. Thus, we need to find -// its starting location. +// Note that we only need to register the start addresses of the object that +// overlays the first address of a card; we need to do this for every card. +// In other words, register_object() checks if the object crosses a card boundary, +// and updates the offset value for each card that the object crosses into. +// For objects that don't straddle cards, nothing needs to be done. // // The RememberedSet template parameter is intended to represent either // ShenandoahDirectCardMarkRememberedSet, or a to-be-implemented @@ -555,26 +441,6 @@ class ShenandoahCardCluster: public CHeapObj { // There is one entry within the object_starts array for each card entry. // - // In the most recent implementation of ShenandoahScanRemembered::process_clusters(), - // there is no need for the get_crossing_object_start() method function, so there is no - // need to maintain the following information. The comment is left in place for now in - // case we find it necessary to add support for this service at a later time. - // - // Bits 0x7fff: If no object starts within this card region, the - // remaining bits of the object_starts array represent - // the absolute word offset within the enclosing - // cluster's memory of the starting address for the - // object that spans the start of this card region's - // memory. If the spanning object begins in memory - // that precedes this card region's cluster, the value - // stored in these bits is the special value 0x7fff. - // (Note that the maximum value required to represent a - // spanning object from within the current cluster is - // ((63 * 64) - 8), which equals 0x0fbf. - // - // In the absence of the need to support get_crossing_object_start(), - // here is discussion of performance: - // // Suppose multiple garbage objects are coalesced during GC sweep // into a single larger "free segment". As each two objects are // coalesced together, the start information pertaining to the second @@ -829,10 +695,30 @@ template class ShenandoahScanRemembered: public CHeapObj { private: - RememberedSet* _rs; ShenandoahCardCluster* _scc; + // Global card stats (cumulative) + HdrSeq _card_stats_scan_rs[MAX_CARD_STAT_TYPE]; + HdrSeq _card_stats_update_refs[MAX_CARD_STAT_TYPE]; + // Per worker card stats (multiplexed by phase) + HdrSeq** _card_stats; + + const char* _card_stats_name[MAX_CARD_STAT_TYPE] = { + "dirty_run", "clean_run", + "dirty_cards", "clean_cards", + "max_dirty_run", "max_clean_run", + "dirty_objs", "clean_objs", + "dirty_scans", "clean_scans", + "alternations" + }; + + const char* _card_stat_log_type[MAX_CARD_STAT_LOG_TYPE] = { + "Scan Remembered Set", "Update Refs" + }; + + int _card_stats_log_counter[2] = {0, 0}; + public: // How to instantiate this object? // ShenandoahDirectCardMarkRememberedSet *rs = @@ -852,10 +738,48 @@ class ShenandoahScanRemembered: public CHeapObj { ShenandoahScanRemembered(RememberedSet *rs) { _rs = rs; _scc = new ShenandoahCardCluster(rs); + + // We allocate ParallelGCThreads worth even though we usually only + // use up to ConcGCThreads, because degenerate collections may employ + // ParallelGCThreads for remembered set scanning. + if (ShenandoahEnableCardStats) { + _card_stats = NEW_C_HEAP_ARRAY(HdrSeq*, ParallelGCThreads, mtGC); + for (uint i = 0; i < ParallelGCThreads; i++) { + _card_stats[i] = new HdrSeq[MAX_CARD_STAT_TYPE]; + } + } else { + _card_stats = nullptr; + } } ~ShenandoahScanRemembered() { delete _scc; + if (ShenandoahEnableCardStats) { + for (uint i = 0; i < ParallelGCThreads; i++) { + delete _card_stats[i]; + } + FREE_C_HEAP_ARRAY(HdrSeq*, _card_stats); + _card_stats = nullptr; + } + assert(_card_stats == nullptr, "Error"); + } + + HdrSeq* card_stats(uint worker_id) { + assert(worker_id < ParallelGCThreads, "Error"); + assert(ShenandoahEnableCardStats == (_card_stats != nullptr), "Error"); + return ShenandoahEnableCardStats ? _card_stats[worker_id] : nullptr; + } + + HdrSeq* card_stats_for_phase(CardStatLogType t) { + switch (t) { + case CARD_STAT_SCAN_RS: + return _card_stats_scan_rs; + case CARD_STAT_UPDATE_REFS: + return _card_stats_update_refs; + default: + guarantee(false, "No such CardStatLogType"); + } + return nullptr; // Quiet compiler } // TODO: We really don't want to share all of these APIs with arbitrary consumers of the ShenandoahScanRemembered abstraction. @@ -923,58 +847,53 @@ class ShenandoahScanRemembered: public CHeapObj { // clear the cards to clean, and clear the object_starts info to no objects void mark_range_as_empty(HeapWord *addr, size_t length_in_words); - // process_clusters() scans a portion of the remembered set during a JVM - // safepoint as part of the root scanning activities that serve to - // initiate concurrent scanning and concurrent evacuation. Multiple - // threads may scan different portions of the remembered set by - // making parallel invocations of process_clusters() with each - // invocation scanning different clusters of the remembered set. + // process_clusters() scans a portion of the remembered set + // to scan roots from old gen into young to identify live objects + // in the young generation. Several worker threads scan different + // portions of the remembered set by making parallel invocations + // of process_clusters() with each invocation scanning different + // "clusters" of the remembered set. // // An invocation of process_clusters() examines all of the - // intergenerational references spanned by count clusters starting - // with first_cluster. The oops argument is assumed to represent a - // thread-local OopClosure into which addresses of intergenerational - // pointer values will be accumulated for the purposes of root scanning. + // intergenerational references spanned by `count` clusters starting + // with `first_cluster`. The `oops` argument is a worker-thread-local + // OopClosure that is applied to all "valid" references in the remembered set. // - // A side effect of executing process_clusters() is to update the card - // table entries, marking dirty cards as clean if they no longer - // hold references to young-gen memory. (THIS IS NOT YET IMPLEMENTED.) + // A side-effect of executing process_clusters() is to update the remembered + // set entries (e.g. marking dirty cards clean if they no longer + // hold references to young-gen memory). // - // The implementation of process_clusters() is designed to efficiently - // minimize work in the large majority of cases for which the - // associated cluster has very few dirty card-table entries. + // An implementation of process_clusters() may choose to efficiently + // address more typical scenarios in the structure of remembered sets. E.g. + // in the generational setting, one might expect remembered sets to be very sparse + // (low mutation rates in the old generation leading to sparse dirty cards, + // each with very few intergenerational pointers). Specific implementations + // may choose to degrade gracefully as the sparsity assumption fails to hold, + // such as when there are sudden spikes in (premature) promotion or in the + // case of an underprovisioned, poorly-tuned, or poorly-shaped heap. // - // At initialization of concurrent marking, invoke process_clusters with - // ClosureType equal to ShenandoahInitMarkRootsClosure. + // At the start of a concurrent young generation marking cycle, we invoke process_clusters + // with ClosureType ShenandoahInitMarkRootsClosure. // - // At initialization of concurrent evacuation, invoke process_clusters with - // ClosureType equal to ShenandoahEvacuateUpdateRootsClosure. + // At the start of a concurrent evacuation phase, we invoke process_clusters with + // ClosureType ShenandoahEvacuateUpdateRootsClosure. - // This is big enough it probably shouldn't be in-lined. On the other hand, there are only a few places this - // code is called from, so it might as well be in-lined. The "real" reason I'm inlining at the moment is because - // the template expansions were making it difficult for the link/loader to resolve references to the template- - // parameterized implementations of this service. + // All template expansions require methods to be defined in the inline.hpp file, but larger + // such methods need not be declared as inline. template - inline void process_clusters(size_t first_cluster, size_t count, HeapWord *end_of_range, ClosureType *oops, bool is_concurrent); + inline void process_clusters(size_t first_cluster, size_t count, HeapWord *end_of_range, ClosureType *oops, bool is_concurrent, uint worker_id); template - inline void process_clusters(size_t first_cluster, size_t count, HeapWord *end_of_range, ClosureType *oops, - bool use_write_table, bool is_concurrent); + void process_clusters(size_t first_cluster, size_t count, HeapWord *end_of_range, ClosureType *oops, + bool use_write_table, bool is_concurrent, uint worker_id); template inline void process_humongous_clusters(ShenandoahHeapRegion* r, size_t first_cluster, size_t count, HeapWord *end_of_range, ClosureType *oops, bool use_write_table, bool is_concurrent); - - template - inline void process_region(ShenandoahHeapRegion* region, ClosureType *cl, bool is_concurrent); - - template - inline void process_region(ShenandoahHeapRegion* region, ClosureType *cl, bool use_write_table, bool is_concurrent); - template inline void process_region_slice(ShenandoahHeapRegion* region, size_t offset, size_t clusters, HeapWord* end_of_range, - ClosureType *cl, bool use_write_table, bool is_concurrent); + ClosureType *cl, bool use_write_table, bool is_concurrent, uint worker_id); // To Do: // Create subclasses of ShenandoahInitMarkRootsClosure and @@ -998,6 +917,18 @@ class ShenandoahScanRemembered: public CHeapObj { // implementations will want to update this value each time they // cross one of these boundaries. void roots_do(OopIterateClosure* cl); + + // Log stats related to card/RS stats for given phase t + void log_card_stats(uint nworkers, CardStatLogType t) PRODUCT_RETURN; +private: + // Log stats for given worker id related into given cumulative card/RS stats + void log_worker_card_stats(uint worker_id, HdrSeq* cum_stats) PRODUCT_RETURN; + + // Log given stats + inline void log_card_stats(HdrSeq* stats) PRODUCT_RETURN; + + // Merge the stats from worked_id into the given summary stats, and clear the worker_id's stats. + void merge_worker_card_stats_cumulative(HdrSeq* worker_stats, HdrSeq* cum_stats) PRODUCT_RETURN; }; struct ShenandoahRegionChunk { @@ -1109,6 +1040,7 @@ class ShenandoahScanRememberedTask : public WorkerTask { ShenandoahReferenceProcessor* _rp; ShenandoahRegionChunkIterator* _work_list; bool _is_concurrent; + public: ShenandoahScanRememberedTask(ShenandoahObjToScanQueueSet* queue_set, ShenandoahObjToScanQueueSet* old_queue_set, diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp index 936882e40476c..214f29e214abb 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp @@ -30,6 +30,7 @@ #include "oops/oop.hpp" #include "oops/objArrayOop.hpp" #include "gc/shared/collectorCounters.hpp" +#include "gc/shenandoah/shenandoahCardStats.hpp" #include "gc/shenandoah/shenandoahCardTable.hpp" #include "gc/shenandoah/shenandoahHeap.hpp" #include "gc/shenandoah/shenandoahHeapRegion.hpp" @@ -463,34 +464,35 @@ ShenandoahScanRemembered::mark_range_as_empty(HeapWord *addr, siz template template -inline void -ShenandoahScanRemembered::process_clusters(size_t first_cluster, size_t count, HeapWord *end_of_range, - ClosureType *cl, bool is_concurrent) { - process_clusters(first_cluster, count, end_of_range, cl, false, is_concurrent); +inline void ShenandoahScanRemembered::process_clusters(size_t first_cluster, size_t count, HeapWord *end_of_range, + ClosureType *cl, bool is_concurrent, uint worker_id) { + process_clusters(first_cluster, count, end_of_range, cl, false, is_concurrent, worker_id); } // Process all objects starting within count clusters beginning with first_cluster for which the start address is -// less than end_of_range. For any such object, process the complete object, even if its end reaches beyond end_of_range. - +// less than end_of_range. For any non-array object whose header lies on a dirty card, scan the entire object, +// even if its end reaches beyond end_of_range. Object arrays, on the other hand,s are precisely dirtied and only +// the portions of the array on dirty cards need to be scanned. +// // Do not CANCEL within process_clusters. It is assumed that if a worker thread accepts responsbility for processing // a chunk of work, it will finish the work it starts. Otherwise, the chunk of work will be lost in the transition to // degenerated execution. template template -inline void -ShenandoahScanRemembered::process_clusters(size_t first_cluster, size_t count, HeapWord *end_of_range, - ClosureType *cl, bool write_table, bool is_concurrent) { +void ShenandoahScanRemembered::process_clusters(size_t first_cluster, size_t count, HeapWord *end_of_range, + ClosureType *cl, bool write_table, bool is_concurrent, uint worker_id) { // Unlike traditional Shenandoah marking, the old-gen resident objects that are examined as part of the remembered set are not - // themselves marked. Each such object will be scanned only once. Any young-gen objects referenced from the remembered set will - // be marked and then subsequently scanned. + // always themselves marked. Each such object will be scanned exactly once. Any young-gen objects referenced from the remembered + // set will be marked and then subsequently scanned. // If old-gen evacuation is active, then MarkingContext for old-gen heap regions is valid. We use the MarkingContext // bits to determine which objects within a DIRTY card need to be scanned. This is necessary because old-gen heap - // regions which are in the candidate collection set have not been coalesced and filled. Thus, these heap regions + // regions that are in the candidate collection set have not been coalesced and filled. Thus, these heap regions // may contain zombie objects. Zombie objects are known to be dead, but have not yet been "collected". Scanning // zombie objects is unsafe because the Klass pointer is not reliable, objects referenced from a zombie may have been - // collected and their memory repurposed, and because zombie objects might refer to objects that are themselves dead. + // collected (if dead), or relocated (if live), or if dead but not yet collected, we don't want to "revive" them + // by marking them (when marking) or evacuating them (when updating refereces). ShenandoahHeap* heap = ShenandoahHeap::heap(); ShenandoahMarkingContext* ctx; @@ -501,26 +503,33 @@ ShenandoahScanRemembered::process_clusters(size_t first_cluster, ctx = nullptr; } - size_t card_index = first_cluster * ShenandoahCardCluster::CardsPerCluster; - HeapWord *start_of_range = _rs->addr_for_card_index(card_index); + size_t cur_cluster = first_cluster; + size_t cur_count = count; + size_t card_index = cur_cluster * ShenandoahCardCluster::CardsPerCluster; + HeapWord* start_of_range = _rs->addr_for_card_index(card_index); ShenandoahHeapRegion* r = heap->heap_region_containing(start_of_range); assert(end_of_range <= r->top(), "process_clusters() examines one region at a time"); - while (count-- > 0) { + NOT_PRODUCT(ShenandoahCardStats stats(ShenandoahCardCluster::CardsPerCluster, card_stats(worker_id));) + + while (cur_count-- > 0) { // TODO: do we want to check cancellation in inner loop, on every card processed? That would be more responsive, // but require more overhead for checking. - card_index = first_cluster * ShenandoahCardCluster::CardsPerCluster; + card_index = cur_cluster * ShenandoahCardCluster::CardsPerCluster; size_t end_card_index = card_index + ShenandoahCardCluster::CardsPerCluster; - first_cluster++; + cur_cluster++; size_t next_card_index = 0; - while (card_index < end_card_index) { + + assert(stats.is_clean(), "Error"); + while (card_index < end_card_index) { // TODO: understand why end_of_range is needed. if (_rs->addr_for_card_index(card_index) > end_of_range) { - count = 0; + cur_count = 0; card_index = end_card_index; break; } bool is_dirty = (write_table)? is_write_card_dirty(card_index): is_card_dirty(card_index); bool has_object = _scc->has_object(card_index); + NOT_PRODUCT(stats.increment_card_cnt(is_dirty);) if (is_dirty) { size_t prev_card_index = card_index; if (has_object) { @@ -546,6 +555,7 @@ ShenandoahScanRemembered::process_clusters(size_t first_cluster, p += start_offset; while (p < endp) { oop obj = cast_to_oop(p); + NOT_PRODUCT(stats.increment_obj_cnt(is_dirty);) // ctx->is_marked() returns true if mark bit set or if obj above TAMS. if (!ctx || ctx->is_marked(obj)) { @@ -557,8 +567,10 @@ ShenandoahScanRemembered::process_clusters(size_t first_cluster, objArrayOop array = objArrayOop(obj); int len = array->length(); array->oop_iterate_range(cl, 0, len); + NOT_PRODUCT(stats.increment_scan_cnt(is_dirty);) } else if (obj->is_instance()) { obj->oop_iterate(cl); + NOT_PRODUCT(stats.increment_scan_cnt(is_dirty);) } else { // Case 3: Primitive array. Do nothing, no oops there. We use the same // performance tweak TypeArrayKlass::oop_oop_iterate_impl is using: @@ -586,78 +598,83 @@ ShenandoahScanRemembered::process_clusters(size_t first_cluster, // Card is dirty but has no object. Card will have been scanned during scan of a previous cluster. card_index++; } - } else if (has_object) { - // Card is clean but has object. - - // Scan the last object that starts within this card memory if it spans at least one dirty card within this cluster - // or if it reaches into the next cluster. - size_t start_offset = _scc->get_last_start(card_index); - HeapWord *card_start = _rs->addr_for_card_index(card_index); - HeapWord *p = card_start + start_offset; - oop obj = cast_to_oop(p); - - size_t last_card; - if (!ctx || ctx->is_marked(obj)) { - HeapWord *nextp = p + obj->size(); - - // Can't use _scc->card_index_for_addr(endp) here because it crashes with assertion - // failure if nextp points to end of heap. Must also not attempt to read past last - // valid index for card table. - last_card = card_index + (nextp - card_start) / CardTable::card_size_in_words(); - last_card = MIN2(last_card, last_valid_index()); - - bool reaches_next_cluster = (last_card > end_card_index); - bool spans_dirty_within_this_cluster = false; - - if (!reaches_next_cluster) { - size_t span_card; - for (span_card = card_index+1; span_card <= last_card; span_card++) - if ((write_table)? _rs->is_write_card_dirty(span_card): _rs->is_card_dirty(span_card)) { - spans_dirty_within_this_cluster = true; - break; + } else { + if (has_object) { + // Card is clean but has object. + // Scan the last object that starts within this card memory if it spans at least one dirty card within this cluster + // or if it reaches into the next cluster. + size_t start_offset = _scc->get_last_start(card_index); + HeapWord *card_start = _rs->addr_for_card_index(card_index); + HeapWord *p = card_start + start_offset; + oop obj = cast_to_oop(p); + + size_t last_card; + if (!ctx || ctx->is_marked(obj)) { + HeapWord *nextp = p + obj->size(); + NOT_PRODUCT(stats.increment_obj_cnt(is_dirty);) + + // Can't use _scc->card_index_for_addr(endp) here because it crashes with assertion + // failure if nextp points to end of heap. Must also not attempt to read past last + // valid index for card table. + last_card = card_index + (nextp - card_start) / CardTable::card_size_in_words(); + last_card = MIN2(last_card, last_valid_index()); + + bool reaches_next_cluster = (last_card > end_card_index); + bool spans_dirty_within_this_cluster = false; + + if (!reaches_next_cluster) { + for (size_t span_card = card_index+1; span_card <= last_card; span_card++) { + if ((write_table)? _rs->is_write_card_dirty(span_card): _rs->is_card_dirty(span_card)) { + spans_dirty_within_this_cluster = true; + break; + } } - } + } + + // TODO: only iterate over this object if it spans dirty within this cluster or within following clusters. + // Code as written is known not to examine a zombie object because either the object is marked, or we are + // not using the mark-context to differentiate objects, so the object is known to have been coalesced and + // filled if it is not "live". - // TODO: only iterate over this object if it spans dirty within this cluster or within following clusters. - // Code as written is known not to examine a zombie object because either the object is marked, or we are - // not using the mark-context to differentiate objects, so the object is known to have been coalesced and - // filled if it is not "live". - - if (reaches_next_cluster || spans_dirty_within_this_cluster) { - if (obj->is_objArray()) { - objArrayOop array = objArrayOop(obj); - int len = array->length(); - array->oop_iterate_range(cl, 0, len); - } else if (obj->is_instance()) { - obj->oop_iterate(cl); + if (reaches_next_cluster || spans_dirty_within_this_cluster) { + if (obj->is_objArray()) { + objArrayOop array = objArrayOop(obj); + int len = array->length(); + array->oop_iterate_range(cl, 0, len); + NOT_PRODUCT(stats.increment_scan_cnt(is_dirty);) + } else if (obj->is_instance()) { + obj->oop_iterate(cl); + NOT_PRODUCT(stats.increment_scan_cnt(is_dirty);) + } else { + // Case 3: Primitive array. Do nothing, no oops there. We use the same + // performance tweak TypeArrayKlass::oop_oop_iterate_impl is using: + // We skip iterating over the klass pointer since we know that + // Universe::TypeArrayKlass never moves. + assert (obj->is_typeArray(), "should be type array"); + } + } + } else { + // The object that spans end of this clean card is not marked, so no need to scan it or its + // unmarked neighbors. Containing region r is initialized above. + HeapWord* tams = ctx->top_at_mark_start(r); + HeapWord* nextp; + if (p >= tams) { + nextp = p + obj->size(); } else { - // Case 3: Primitive array. Do nothing, no oops there. We use the same - // performance tweak TypeArrayKlass::oop_oop_iterate_impl is using: - // We skip iterating over the klass pointer since we know that - // Universe::TypeArrayKlass never moves. - assert (obj->is_typeArray(), "should be type array"); + nextp = ctx->get_next_marked_addr(p, tams); } + last_card = card_index + (nextp - card_start) / CardTable::card_size_in_words(); } + // Increment card_index to account for the spanning object, even if we didn't scan it. + card_index = (last_card > card_index)? last_card: card_index + 1; } else { - // The object that spans end of this clean card is not marked, so no need to scan it or its - // unmarked neighbors. Containing region r is initialized above. - HeapWord* tams = ctx->top_at_mark_start(r); - HeapWord* nextp; - if (p >= tams) { - nextp = p + obj->size(); - } else { - nextp = ctx->get_next_marked_addr(p, tams); - } - last_card = card_index + (nextp - card_start) / CardTable::card_size_in_words(); + // Card is clean and has no object. No need to clean this card. + card_index++; } - // Increment card_index to account for the spanning object, even if we didn't scan it. - card_index = (last_card > card_index)? last_card: card_index + 1; - } else { - // Card is clean and has no object. No need to clean this card. - card_index++; } - } - } + } // end of a range of cards in current cluster + NOT_PRODUCT(stats.update_run(true /* record */);) + } // end of all clusters } // Given that this range of clusters is known to span a humongous object spanned by region r, scan the @@ -681,30 +698,12 @@ ShenandoahScanRemembered::process_humongous_clusters(ShenandoahHe start_region->oop_iterate_humongous_slice(cl, true, first_cluster_addr, spanned_words, write_table, is_concurrent); } -template -template -inline void -ShenandoahScanRemembered::process_region(ShenandoahHeapRegion *region, ClosureType *cl, bool is_concurrent) { - process_region(region, cl, false, is_concurrent); -} - -template -template -inline void -ShenandoahScanRemembered::process_region(ShenandoahHeapRegion *region, ClosureType *cl, - bool use_write_table, bool is_concurrent) { - size_t cluster_size = - CardTable::card_size_in_words() * ShenandoahCardCluster::CardsPerCluster; - size_t clusters = ShenandoahHeapRegion::region_size_words() / cluster_size; - process_region_slice(region, 0, clusters, region->end(), cl, use_write_table, is_concurrent); -} - template template inline void ShenandoahScanRemembered::process_region_slice(ShenandoahHeapRegion *region, size_t start_offset, size_t clusters, HeapWord *end_of_range, ClosureType *cl, bool use_write_table, - bool is_concurrent) { + bool is_concurrent, uint worker_id) { HeapWord *start_of_range = region->bottom() + start_offset; size_t cluster_size = CardTable::card_size_in_words() * ShenandoahCardCluster::CardsPerCluster; @@ -753,7 +752,7 @@ ShenandoahScanRemembered::process_region_slice(ShenandoahHeapRegi ShenandoahHeapRegion* start_region = region->humongous_start_region(); process_humongous_clusters(start_region, start_cluster_no, clusters, end_of_range, cl, use_write_table, is_concurrent); } else { - process_clusters(start_cluster_no, clusters, end_of_range, cl, use_write_table, is_concurrent); + process_clusters(start_cluster_no, clusters, end_of_range, cl, use_write_table, is_concurrent, worker_id); } } } @@ -775,7 +774,7 @@ ShenandoahScanRemembered::addr_for_cluster(size_t cluster_no) { // This is used only for debug verification so don't worry about making the scan parallel. template -inline void ShenandoahScanRemembered::roots_do(OopIterateClosure* cl) { +void ShenandoahScanRemembered::roots_do(OopIterateClosure* cl) { ShenandoahHeap* heap = ShenandoahHeap::heap(); for (size_t i = 0, n = heap->num_regions(); i < n; ++i) { ShenandoahHeapRegion* region = heap->get_region(i); @@ -793,12 +792,64 @@ inline void ShenandoahScanRemembered::roots_do(OopIterateClosure* process_humongous_clusters(region->humongous_start_region(), start_cluster_no, num_clusters, end_of_range, cl, false /* is_write_table */, false /* is_concurrent */); } else { - process_clusters(start_cluster_no, num_clusters, end_of_range, cl, false /* is_concurrent */); + process_clusters(start_cluster_no, num_clusters, end_of_range, cl, false /* is_concurrent */, 0); } } } } +#ifndef PRODUCT +// Log given card stats +template +inline void ShenandoahScanRemembered::log_card_stats(HdrSeq* stats) { + for (int i = 0; i < MAX_CARD_STAT_TYPE; i++) { + log_info(gc, remset)("%18s: [ %8.2f %8.2f %8.2f %8.2f %8.2f ]", + _card_stats_name[i], + stats[i].percentile(0), stats[i].percentile(25), + stats[i].percentile(50), stats[i].percentile(75), + stats[i].maximum()); + } +} + +// Log card stats for all nworkers for a specific phase t +template +void ShenandoahScanRemembered::log_card_stats(uint nworkers, CardStatLogType t) { + assert(ShenandoahEnableCardStats, "Do not call"); + HdrSeq* cum_stats = card_stats_for_phase(t); + log_info(gc, remset)("%s", _card_stat_log_type[t]); + for (uint i = 0; i < nworkers; i++) { + log_worker_card_stats(i, cum_stats); + } + + // Every so often, log the cumulative global stats + if (++_card_stats_log_counter[t] >= ShenandoahCardStatsLogInterval) { + _card_stats_log_counter[t] = 0; + log_info(gc, remset)("Cumulative stats"); + log_card_stats(cum_stats); + } +} + +// Log card stats for given worker_id, & clear them after merging into given cumulative stats +template +void ShenandoahScanRemembered::log_worker_card_stats(uint worker_id, HdrSeq* cum_stats) { + assert(ShenandoahEnableCardStats, "Do not call"); + + HdrSeq* worker_card_stats = card_stats(worker_id); + log_info(gc, remset)("Worker %u Card Stats: ", worker_id); + log_card_stats(worker_card_stats); + // Merge worker stats into the cumulative stats & clear worker stats + merge_worker_card_stats_cumulative(worker_card_stats, cum_stats); +} + +template +void ShenandoahScanRemembered::merge_worker_card_stats_cumulative( + HdrSeq* worker_stats, HdrSeq* cum_stats) { + for (int i = 0; i < MAX_CARD_STAT_TYPE; i++) { + worker_stats[i].merge(cum_stats[i]); + } +} +#endif + inline bool ShenandoahRegionChunkIterator::has_next() const { return _index < _total_chunks; } @@ -838,5 +889,4 @@ inline bool ShenandoahRegionChunkIterator::next(struct ShenandoahRegionChunk *as assignment->_chunk_size = group_chunk_size; return true; } - #endif // SHARE_GC_SHENANDOAH_SHENANDOAHSCANREMEMBEREDINLINE_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp index e8e9fd8172cd2..f92871240bce8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp @@ -540,7 +540,14 @@ \ product(uintx, ShenandoahAgingCyclePeriod, 1, EXPERIMENTAL, \ "With generational mode, increment the age of objects and" \ - "regions each time this many young-gen GC cycles are completed.") - // end of GC_SHENANDOAH_FLAGS + "regions each time this many young-gen GC cycles are completed.") \ + \ + notproduct(bool, ShenandoahEnableCardStats, trueInDebug, \ + "Enable statistics collection related to clean & dirty cards") \ + \ + notproduct(int, ShenandoahCardStatsLogInterval, 50, \ + "Log cumulative card stats every so many remembered set or " \ + "update refs scans") \ + // end of GC_SHENANDOAH_FLAGS #endif // SHARE_GC_SHENANDOAH_SHENANDOAH_GLOBALS_HPP diff --git a/src/hotspot/share/utilities/numberSeq.cpp b/src/hotspot/share/utilities/numberSeq.cpp index 0fd9b8612a992..ffc7d4d63755e 100644 --- a/src/hotspot/share/utilities/numberSeq.cpp +++ b/src/hotspot/share/utilities/numberSeq.cpp @@ -110,7 +110,6 @@ double AbsSeq::dsd() const { return sqrt(var); } - void AbsSeq::merge(AbsSeq& abs2, bool clear_this) { if (num() == 0) return; // nothing to do @@ -121,8 +120,7 @@ void AbsSeq::merge(AbsSeq& abs2, bool clear_this) { // Decaying stats need a bit more thought assert(abs2._alpha == _alpha, "Caution: merge incompatible?"); - - // Until JDK-... is fixed, we taint the decaying statistics + // Until JDK-8298902 is fixed, we taint the decaying statistics if (abs2._davg != NAN) { abs2._davg = NAN; abs2._dvariance = NAN; From ba808494675509a7ed2d97d08a9fbc971dbc0900 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Wed, 4 Jan 2023 16:51:06 +0000 Subject: [PATCH 168/254] Allow heuristic trigger to increase capacity instead of running a collection Reviewed-by: kdnilsen --- .../shenandoahAdaptiveHeuristics.cpp | 28 +++++++++++++++-- .../shenandoahAdaptiveHeuristics.hpp | 2 ++ .../heuristics/shenandoahHeuristics.cpp | 4 +++ .../heuristics/shenandoahHeuristics.hpp | 2 ++ .../heuristics/shenandoahOldHeuristics.cpp | 11 +++++++ .../heuristics/shenandoahOldHeuristics.hpp | 2 ++ .../gc/shenandoah/shenandoahGeneration.cpp | 10 ++++-- .../gc/shenandoah/shenandoahGeneration.hpp | 7 +++++ .../gc/shenandoah/shenandoahMmuTracker.cpp | 31 ++++++++++++------- .../gc/shenandoah/shenandoahMmuTracker.hpp | 15 +++++---- 10 files changed, 88 insertions(+), 24 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp index a22d64c655628..0a039b2c31e37 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp @@ -360,7 +360,7 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { _generation->name(), byte_size_in_proper_unit(allocation_headroom), proper_unit_for_byte_size(allocation_headroom), byte_size_in_proper_unit(min_threshold), proper_unit_for_byte_size(min_threshold)); - return true; + return resize_and_evaluate(); } // Check if we need to learn a bit about the application @@ -458,7 +458,7 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { byte_size_in_proper_unit(allocation_headroom), proper_unit_for_byte_size(allocation_headroom)); _last_trigger = RATE; - return true; + return resize_and_evaluate(); } bool is_spiking = _allocation_rate.is_spiking(rate, _spike_threshold_sd); @@ -470,12 +470,34 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { _spike_threshold_sd); _last_trigger = SPIKE; - return true; + return resize_and_evaluate(); } return ShenandoahHeuristics::should_start_gc(); } +bool ShenandoahAdaptiveHeuristics::resize_and_evaluate() { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + if (!heap->mode()->is_generational()) { + // We only attempt to resize the generations in generational mode. + return true; + } + + if (_gc_times_learned < ShenandoahLearningSteps) { + // We aren't going to attempt to resize our generation until we have 'learned' + // something about it. This provides a kind of cool down period after we've made + // a change, to help prevent thrashing. + return true; + } + + if (!heap->generation_sizer()->transfer_capacity(_generation)) { + // We could not enlarge our generation, so we must start a gc cycle. + return true; + } + + return should_start_gc(); +} + void ShenandoahAdaptiveHeuristics::adjust_last_trigger_parameters(double amount) { switch (_last_trigger) { case RATE: diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp index 45655ef388ad7..7c5f710c04e37 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp @@ -99,6 +99,8 @@ class ShenandoahAdaptiveHeuristics : public ShenandoahHeuristics { void adjust_margin_of_error(double amount); void adjust_spike_threshold(double amount); + bool resize_and_evaluate(); + ShenandoahAllocationRate _allocation_rate; // The margin of error expressed in standard deviations to add to our diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp index 9b31e4a01fbcd..a26cb7919183c 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp @@ -330,6 +330,10 @@ void ShenandoahHeuristics::record_allocation_failure_gc() { void ShenandoahHeuristics::record_requested_gc() { // Assume users call System.gc() when external state changes significantly, // which forces us to re-learn the GC timings and allocation rates. + reset_gc_learning(); +} + +void ShenandoahHeuristics::reset_gc_learning() { _gc_times_learned = 0; } diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp index dcafb3424777a..380348fd966bf 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp @@ -155,6 +155,8 @@ class ShenandoahHeuristics : public CHeapObj { virtual void record_requested_gc(); + virtual void reset_gc_learning(); + virtual size_t select_aged_regions(size_t old_available, size_t num_regions, bool preselected_regions[]); virtual void choose_collection_set(ShenandoahCollectionSet* collection_set, ShenandoahOldHeuristics* old_heuristics); diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp index 45d1a43d61680..e7a8b304445cb 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp @@ -331,6 +331,12 @@ void ShenandoahOldHeuristics::abandon_collection_candidates() { } void ShenandoahOldHeuristics::handle_promotion_failure() { + if (!_promotion_failed) { + if (ShenandoahHeap::heap()->generation_sizer()->transfer_capacity(_old_generation)) { + log_info(gc)("Increased size of old generation due to promotion failure."); + } + // TODO: Increase tenuring threshold to push back on promotions. + } _promotion_failed = true; } @@ -387,6 +393,10 @@ void ShenandoahOldHeuristics::record_requested_gc() { _trigger_heuristic->record_requested_gc(); } +void ShenandoahOldHeuristics::reset_gc_learning() { + _trigger_heuristic->reset_gc_learning(); +} + bool ShenandoahOldHeuristics::can_unload_classes() { return _trigger_heuristic->can_unload_classes(); } @@ -419,3 +429,4 @@ void ShenandoahOldHeuristics::choose_collection_set_from_regiondata(ShenandoahCo ShouldNotReachHere(); } + diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp index f4384f5aa43c1..3fc5c7f2ff2b9 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp @@ -149,6 +149,8 @@ class ShenandoahOldHeuristics : public ShenandoahHeuristics { virtual void record_requested_gc() override; + virtual void reset_gc_learning() override; + virtual bool can_unload_classes() override; virtual bool can_unload_classes_normal() override; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index 67ed1f0e4115d..f3cc8e0d7a5d6 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -273,7 +273,7 @@ void ShenandoahGeneration::compute_evacuation_budgets(ShenandoahHeap* heap, bool heap->set_old_evac_reserve(old_evacuation_reserve); heap->reset_old_evac_expended(); - // Compute the young evauation reserve: This is how much memory is available for evacuating young-gen objects. + // Compute the young evacuation reserve: This is how much memory is available for evacuating young-gen objects. // We ignore the possible effect of promotions, which reduce demand for young-gen evacuation memory. // // TODO: We could give special treatment to the regions that have reached promotion age, because we know their @@ -288,7 +288,7 @@ void ShenandoahGeneration::compute_evacuation_budgets(ShenandoahHeap* heap, bool // 1. (young_gen->capacity() * ShenandoahEvacReserve) / 100 // 2. (young_gen->available() + old_gen_memory_available_to_be_loaned // - // ShenandoahEvacReserve represents the configured taget size of the evacuation region. We can only honor + // ShenandoahEvacReserve represents the configured target size of the evacuation region. We can only honor // this target if there is memory available to hold the evacuations. Memory is available if it is already // free within young gen, or if it can be borrowed from old gen. Since we have not yet chosen the collection // sets, we do not yet know the exact accounting of how many regions will be freed by this collection pass. @@ -992,12 +992,13 @@ size_t ShenandoahGeneration::adjusted_unaffiliated_regions() const { return (adjusted_capacity() - used_regions_size()) / ShenandoahHeapRegion::region_size_bytes(); } - void ShenandoahGeneration::increase_capacity(size_t increment) { shenandoah_assert_heaplocked_or_safepoint(); assert(_max_capacity + increment <= ShenandoahHeap::heap()->max_size_for(this), "Cannot increase generation capacity beyond maximum."); _max_capacity += increment; _soft_max_capacity += increment; + _adjusted_capacity += increment; + heuristics()->reset_gc_learning(); } void ShenandoahGeneration::decrease_capacity(size_t decrement) { @@ -1005,6 +1006,8 @@ void ShenandoahGeneration::decrease_capacity(size_t decrement) { assert(_max_capacity - decrement >= ShenandoahHeap::heap()->min_size_for(this), "Cannot decrease generation capacity beyond minimum."); _max_capacity -= decrement; _soft_max_capacity -= decrement; + _adjusted_capacity -= decrement; + heuristics()->reset_gc_learning(); } void ShenandoahGeneration::record_success_concurrent(bool abbreviated) { @@ -1018,6 +1021,7 @@ void ShenandoahGeneration::record_success_degenerated() { } void ShenandoahGeneration::add_collection_time(double time_seconds) { + shenandoah_assert_control_or_vm_thread(); _collection_thread_time_s += time_seconds; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index 52882ba84196b..c5134853b0796 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -73,6 +73,10 @@ class ShenandoahGeneration : public CHeapObj { ShenandoahGeneration(GenerationMode generation_mode, uint max_workers, size_t max_capacity, size_t soft_max_capacity); ~ShenandoahGeneration(); + bool is_young() const { return _generation_mode == YOUNG; } + bool is_old() const { return _generation_mode == OLD; } + bool is_global() const { return _generation_mode == GLOBAL; } + inline GenerationMode generation_mode() const { return _generation_mode; } inline ShenandoahHeuristics* heuristics() const { return _heuristics; } @@ -111,6 +115,9 @@ class ShenandoahGeneration : public CHeapObj { void reset_bytes_allocated_since_gc_start(); void increase_allocated(size_t bytes); + // Changing the size of the generation will reset the times learned for the heuristic. The heuristic will need to + // relearn collection performance metrics. This also has the effect of preventing further capacity changes from the + // heuristics until at least ShenandoahLearningSteps(5) number of cycles has completed. void increase_capacity(size_t increment); void decrease_capacity(size_t decrement); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp index 34835db1435b6..987e3339777fb 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp @@ -36,10 +36,10 @@ class ShenandoahMmuTask : public PeriodicTask { ShenandoahMmuTracker* _mmu_tracker; public: - ShenandoahMmuTask(ShenandoahMmuTracker* mmu_tracker) : + explicit ShenandoahMmuTask(ShenandoahMmuTracker* mmu_tracker) : PeriodicTask(GCPauseIntervalMillis), _mmu_tracker(mmu_tracker) {} - virtual void task() override { + void task() override { _mmu_tracker->report(); } }; @@ -48,7 +48,7 @@ class ThreadTimeAccumulator : public ThreadClosure { public: size_t total_time; ThreadTimeAccumulator() : total_time(0) {} - virtual void do_thread(Thread* thread) override { + void do_thread(Thread* thread) override { total_time += os::thread_cpu_time(thread); } }; @@ -73,6 +73,7 @@ double ShenandoahMmuTracker::process_time_seconds() { ShenandoahMmuTracker::ShenandoahMmuTracker() : _generational_reference_time_s(0.0), _process_reference_time_s(0.0), + _collector_reference_time_s(0.0), _mmu_periodic_task(new ShenandoahMmuTask(this)), _mmu_average(10, ShenandoahAdaptiveDecayFactor) { } @@ -120,7 +121,7 @@ ShenandoahGenerationSizer::ShenandoahGenerationSizer(ShenandoahMmuTracker* mmu_t _use_adaptive_sizing(true), _min_desired_young_size(0), _max_desired_young_size(0), - _resize_increment(YoungGenerationSizeIncrement / 100.0), + _resize_increment(double(YoungGenerationSizeIncrement) / 100.0), _mmu_tracker(mmu_tracker) { if (FLAG_IS_CMDLINE(NewRatio)) { @@ -201,7 +202,7 @@ void ShenandoahGenerationSizer::heap_size_changed(size_t heap_size) { recalculate_min_max_young_length(heap_size); } -bool ShenandoahGenerationSizer::adjust_generation_sizes() { +bool ShenandoahGenerationSizer::adjust_generation_sizes() const { shenandoah_assert_generational(); if (!use_adaptive_sizing()) { return false; @@ -238,7 +239,17 @@ bool ShenandoahGenerationSizer::adjust_generation_sizes() { } } -bool ShenandoahGenerationSizer::transfer_capacity(ShenandoahGeneration* from, ShenandoahGeneration* to) { +bool ShenandoahGenerationSizer::transfer_capacity(ShenandoahGeneration* target) const { + ShenandoahHeapLocker locker(ShenandoahHeap::heap()->lock()); + if (target->is_young()) { + return transfer_capacity(ShenandoahHeap::heap()->old_generation(), target); + } else { + assert(target->is_old(), "Expected old generation, if not young."); + return transfer_capacity(ShenandoahHeap::heap()->young_generation(), target); + } +} + +bool ShenandoahGenerationSizer::transfer_capacity(ShenandoahGeneration* from, ShenandoahGeneration* to) const { shenandoah_assert_heaplocked_or_safepoint(); size_t available_regions = from->free_unaffiliated_regions(); @@ -270,10 +281,6 @@ bool ShenandoahGenerationSizer::transfer_capacity(ShenandoahGeneration* from, Sh return true; } -size_t round_down_to_multiple_of_region_size(size_t bytes) { - return (bytes / ShenandoahHeapRegion::region_size_bytes()) * ShenandoahHeapRegion::region_size_bytes(); -} - size_t ShenandoahGenerationSizer::adjust_transfer_from_young(ShenandoahGeneration* from, size_t bytes_to_transfer) const { assert(from->generation_mode() == YOUNG, "Expect to transfer from young"); size_t new_young_size = from->max_capacity() - bytes_to_transfer; @@ -283,7 +290,7 @@ size_t ShenandoahGenerationSizer::adjust_transfer_from_young(ShenandoahGeneratio assert(minimum_size <= from->max_capacity(), "Young is under minimum capacity."); // If the transfer violates the minimum size and there is still some capacity to transfer, // adjust the transfer to take the size to the minimum. Note that this may be zero. - bytes_to_transfer = round_down_to_multiple_of_region_size(from->max_capacity() - minimum_size); + bytes_to_transfer = align_down(from->max_capacity() - minimum_size, ShenandoahHeapRegion::region_size_bytes()); } return bytes_to_transfer; } @@ -297,7 +304,7 @@ size_t ShenandoahGenerationSizer::adjust_transfer_to_young(ShenandoahGeneration* assert(maximum_size >= to->max_capacity(), "Young is over maximum capacity"); // If the transfer violates the maximum size and there is still some capacity to transfer, // adjust the transfer to take the size to the maximum. Note that this may be zero. - bytes_to_transfer = round_down_to_multiple_of_region_size(maximum_size - to->max_capacity()); + bytes_to_transfer = align_down(maximum_size - to->max_capacity(), ShenandoahHeapRegion::region_size_bytes()); } return bytes_to_transfer; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.hpp index 113d66442e22b..f322bb0500f99 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.hpp @@ -114,17 +114,16 @@ class ShenandoahGenerationSizer { // given the number of heap regions depending on the kind of sizing algorithm. void recalculate_min_max_young_length(size_t heap_size); - // This will attempt to transfer capacity from one generation to the other. It - // returns true if a transfer is made, false otherwise. - bool transfer_capacity(ShenandoahGeneration* from, ShenandoahGeneration* to); - // These two methods are responsible for enforcing the minimum and maximum // constraints for the size of the generations. size_t adjust_transfer_from_young(ShenandoahGeneration* from, size_t bytes_to_transfer) const; size_t adjust_transfer_to_young(ShenandoahGeneration* to, size_t bytes_to_transfer) const; + // This will attempt to transfer capacity from one generation to the other. It + // returns true if a transfer is made, false otherwise. + bool transfer_capacity(ShenandoahGeneration* from, ShenandoahGeneration* to) const; public: - ShenandoahGenerationSizer(ShenandoahMmuTracker* mmu_tracker); + explicit ShenandoahGenerationSizer(ShenandoahMmuTracker* mmu_tracker); // Calculate the maximum length of the young gen given the number of regions // depending on the sizing algorithm. @@ -149,7 +148,11 @@ class ShenandoahGenerationSizer { // The minimum and maximum sizes of the young generation are controlled by // ShenandoahMinYoungPercentage and ShenandoahMaxYoungPercentage, respectively. // The method returns true when an adjustment is made, false otherwise. - bool adjust_generation_sizes(); + bool adjust_generation_sizes() const; + + // This may be invoked by a heuristic (from regulator thread) before it + // decides to run a collection. + bool transfer_capacity(ShenandoahGeneration* target) const; }; #endif //SHARE_GC_SHENANDOAH_SHENANDOAHMMUTRACKER_HPP From 6daaa75a34857998be5ac4dd53bdf0db289fd3a1 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Wed, 4 Jan 2023 23:22:42 +0000 Subject: [PATCH 169/254] Enforce that generation sizes align with region sizes Reviewed-by: ysr, wkemper --- src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp | 3 +++ src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp | 2 ++ 2 files changed, 5 insertions(+) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index f3cc8e0d7a5d6..288dcec8f292c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -989,6 +989,9 @@ size_t ShenandoahGeneration::adjusted_capacity() const { size_t ShenandoahGeneration::adjusted_unaffiliated_regions() const { assert(adjusted_capacity() >= used_regions_size(), "adjusted_unaffiliated_regions() cannot return negative"); + assert((adjusted_capacity() - used_regions_size()) % ShenandoahHeapRegion::region_size_bytes() == 0, + "adjusted_capacity (" SIZE_FORMAT ") and used regions size (" SIZE_FORMAT ") should be multiples of region_size_bytes", + adjusted_capacity(), used_regions_size()); return (adjusted_capacity() - used_regions_size()) / ShenandoahHeapRegion::region_size_bytes(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp index 987e3339777fb..ea87186af1c8b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp @@ -160,11 +160,13 @@ ShenandoahGenerationSizer::ShenandoahGenerationSizer(ShenandoahMmuTracker* mmu_t size_t ShenandoahGenerationSizer::calculate_min_size(size_t heap_size) { size_t default_value = (heap_size * ShenandoahMinYoungPercentage) / 100; + default_value &= ~ShenandoahHeapRegion::region_size_bytes_mask(); return MAX2(ShenandoahHeapRegion::region_size_bytes(), default_value); } size_t ShenandoahGenerationSizer::calculate_max_size(size_t heap_size) { size_t default_value = (heap_size * ShenandoahMaxYoungPercentage) / 100; + default_value &= ~ShenandoahHeapRegion::region_size_bytes_mask(); return MAX2(ShenandoahHeapRegion::region_size_bytes(), default_value); } From 7e9a1d49eae7bff80e2b678f2402a4ecdf6c748f Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Thu, 5 Jan 2023 17:01:15 +0000 Subject: [PATCH 170/254] Fix allocate aligned Reviewed-by: wkemper, ysr --- .../share/gc/shenandoah/shenandoahFreeSet.cpp | 51 +++++++++++++++---- .../gc/shenandoah/shenandoahGeneration.cpp | 2 +- .../shenandoahHeapRegion.inline.hpp | 21 +++++--- .../gc/shenandoah/shenandoahVerifier.cpp | 2 - 4 files changed, 56 insertions(+), 20 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index 401611692560e..ba1120ab5d324 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -197,9 +197,7 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah r->is_trash()) { return NULL; } - try_recycle_trashed(r); - if (r->affiliation() == ShenandoahRegionAffiliation::FREE) { ShenandoahMarkingContext* const ctx = _heap->complete_marking_context(); r->set_affiliation(req.affiliation()); @@ -227,7 +225,6 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah } in_new_region = r->is_empty(); - HeapWord* result = NULL; size_t size = req.size(); @@ -239,15 +236,41 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah size_t usable_free = (free / CardTable::card_size()) << CardTable::card_shift(); if ((free != usable_free) && (free - usable_free < ShenandoahHeap::min_fill_size() * HeapWordSize)) { // We'll have to add another card's memory to the padding - usable_free -= CardTable::card_size(); + if (usable_free > CardTable::card_size()) { + usable_free -= CardTable::card_size(); + } else { + assert(usable_free == 0, "usable_free is a multiple of card_size and card_size > min_fill_size"); + } } free /= HeapWordSize; usable_free /= HeapWordSize; + size_t remnant = size % CardTable::card_size_in_words(); + if (remnant > 0) { + // Since we have Elastic TLABs, align size up. This is consistent with aligning min_size up. + size = size - remnant + CardTable::card_size_in_words(); + } if (size > usable_free) { size = usable_free; + assert(size % CardTable::card_size_in_words() == 0, "usable_free is a multiple of card table size"); } - if (size >= req.min_size()) { + + size_t adjusted_min_size = req.min_size(); + remnant = adjusted_min_size % CardTable::card_size_in_words(); + if (remnant > 0) { + // Round up adjusted_min_size to a multiple of alignment size + adjusted_min_size = adjusted_min_size - remnant + CardTable::card_size_in_words(); + } + if (size >= adjusted_min_size) { result = r->allocate_aligned(size, req, CardTable::card_size()); + // TODO: Fix allocate_aligned() to provide min_size() allocation if insufficient memory for desired size. + // Then add: assert(result != nullptr, "Allocation cannot fail"); + assert(r->top() <= r->end(), "Allocation cannot span end of region"); + // actual_size() will be set to size below. + assert((result == nullptr) || (size % CardTable::card_size_in_words() == 0), + "PLAB size must be multiple of card size"); + assert((result == nullptr) || (((uintptr_t) result) % CardTable::card_size_in_words() == 0), + "PLAB start must align with card boundary"); + if (result != nullptr && free > usable_free) { // Account for the alignment padding size_t padding = (free - usable_free) * HeapWordSize; @@ -258,6 +281,7 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah _heap->increase_used(padding); } } + // Otherwise, leave result == NULL because the adjusted size is smaller than min size. } else { // This is a GCLAB or a TLAB allocation size_t free = align_down(r->free() >> LogHeapWordSize, MinObjAlignment); @@ -279,13 +303,20 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah usable_free /= HeapWordSize; if ((free != usable_free) && (free - usable_free < ShenandoahHeap::min_fill_size() * HeapWordSize)) { // We'll have to add another card's memory to the padding - usable_free -= CardTable::card_size(); + if (usable_free > CardTable::card_size_in_words()) { + usable_free -= CardTable::card_size_in_words(); + } else { + assert(usable_free == 0, "usable_free is a multiple of card_size and card_size > min_fill_size"); + } } + assert(size % CardTable::card_size_in_words() == 0, "PLAB size must be multiple of remembered set card size"); if (size <= usable_free) { - assert(size % CardTable::card_size_in_words() == 0, "PLAB size must be multiple of remembered set card size"); - result = r->allocate_aligned(size, req, CardTable::card_size()); - if (result != nullptr) { + assert(result != nullptr, "Allocation cannot fail"); + assert(r->top() <= r->end(), "Allocation cannot span end of region"); + assert(req.actual_size() % CardTable::card_size_in_words() == 0, "PLAB start must align with card boundary"); + assert(((uintptr_t) result) % CardTable::card_size_in_words() == 0, "PLAB start must align with card boundary"); + if (free > usable_free) { // Account for the alignment padding size_t padding = (free - usable_free) * HeapWordSize; increase_used(padding); @@ -309,7 +340,7 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah _heap->young_generation()->increase_used(size * HeapWordSize); increase_used(size * HeapWordSize); } else { - // assert(req.is_gc_alloc(), "Should be gc_alloc since req wasn't mutator alloc"); + assert(req.is_gc_alloc(), "Should be gc_alloc since req wasn't mutator alloc"); // For GC allocations, we advance update_watermark because the objects relocated into this memory during // evacuation are not updated during evacuation. For both young and old regions r, it is essential that all diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index 288dcec8f292c..66c0ae45c1c47 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -990,7 +990,7 @@ size_t ShenandoahGeneration::adjusted_capacity() const { size_t ShenandoahGeneration::adjusted_unaffiliated_regions() const { assert(adjusted_capacity() >= used_regions_size(), "adjusted_unaffiliated_regions() cannot return negative"); assert((adjusted_capacity() - used_regions_size()) % ShenandoahHeapRegion::region_size_bytes() == 0, - "adjusted_capacity (" SIZE_FORMAT ") and used regions size (" SIZE_FORMAT ") should be multiples of region_size_bytes", + "adjusted capacity (" SIZE_FORMAT ") and used regions size (" SIZE_FORMAT ") should be multiples of region_size_bytes", adjusted_capacity(), used_regions_size()); return (adjusted_capacity() - used_regions_size()) / ShenandoahHeapRegion::region_size_bytes(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp index d6c38e6caf60a..a37e14a0596f5 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp @@ -44,18 +44,25 @@ HeapWord* ShenandoahHeapRegion::allocate_aligned(size_t size, ShenandoahAllocReq uintptr_t addr_as_int = (uintptr_t) obj; size_t unalignment_bytes = addr_as_int % alignment_in_bytes; - size_t unalignment_words = unalignment_bytes / HeapWordSize; - if (pointer_delta(end(), obj + unalignment_words) >= size) { - if (unalignment_words > 0) { - size_t pad_words = (alignment_in_bytes / HeapWordSize) - unalignment_words; - if (pad_words < ShenandoahHeap::min_fill_size()) { - pad_words += (alignment_in_bytes / HeapWordSize); - } + assert(unalignment_bytes % HeapWordSize == 0, "top should be multiple of HeapWordSize"); + + size_t pad_words = 0; + if (unalignment_bytes > 0) { + pad_words = (alignment_in_bytes - unalignment_bytes) / HeapWordSize; + } + if ((pad_words > 0) && (pad_words < ShenandoahHeap::min_fill_size())) { + pad_words += alignment_in_bytes / HeapWordSize; + } + if (pointer_delta(end(), obj + pad_words) >= size) { + if (pad_words > 0) { ShenandoahHeap::fill_with_object(obj, pad_words); + // register the filled pad object ShenandoahHeap::heap()->card_scan()->register_object(obj); obj += pad_words; } + // We don't need to register the PLAB. Its content will be registered as objects are allocated within it and/or + // when the PLAB is retired. make_regular_allocation(req.affiliation()); adjust_alloc_metadata(req.type(), size); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp index f7288601df4bf..82f44b7a738a1 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp @@ -792,9 +792,7 @@ void ShenandoahVerifier::verify_at_safepoint(const char* label, ShenandoahCalculateRegionStatsClosure cl; _heap->heap_region_iterate(&cl); - size_t heap_used = _heap->used(); - guarantee(cl.used() == heap_used, "%s: heap used size must be consistent: heap-used = " SIZE_FORMAT "%s, regions-used = " SIZE_FORMAT "%s", label, From cb70d299998937138c03a2a7558fe1f6f3cdba0e Mon Sep 17 00:00:00 2001 From: William Kemper Date: Thu, 5 Jan 2023 23:02:39 +0000 Subject: [PATCH 171/254] Fix use of uninitialized double Reviewed-by: kdnilsen, ysr --- src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index 66c0ae45c1c47..94d78514e7ece 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -873,6 +873,7 @@ ShenandoahGeneration::ShenandoahGeneration(GenerationMode generation_mode, _generation_mode(generation_mode), _task_queues(new ShenandoahObjToScanQueueSet(max_workers)), _ref_processor(new ShenandoahReferenceProcessor(MAX2(max_workers, 1U))), + _collection_thread_time_s(0.0), _affiliated_region_count(0), _used(0), _bytes_allocated_since_gc_start(0), _max_capacity(max_capacity), _soft_max_capacity(soft_max_capacity), _adjusted_capacity(soft_max_capacity), _heuristics(nullptr) { From c5774a09b35a01c5ab52831a63072d0b753afd64 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Fri, 6 Jan 2023 21:43:34 +0000 Subject: [PATCH 172/254] Plab fallback to minsize Reviewed-by: wkemper, ysr --- .../share/gc/shenandoah/shenandoahFreeSet.cpp | 20 ++++-- .../gc/shenandoah/shenandoahHeapRegion.hpp | 2 +- .../shenandoahHeapRegion.inline.hpp | 65 +++++++++++++------ 3 files changed, 58 insertions(+), 29 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index ba1120ab5d324..3f6efc8520014 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -262,16 +262,15 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah } if (size >= adjusted_min_size) { result = r->allocate_aligned(size, req, CardTable::card_size()); - // TODO: Fix allocate_aligned() to provide min_size() allocation if insufficient memory for desired size. - // Then add: assert(result != nullptr, "Allocation cannot fail"); + assert(result != nullptr, "Allocation cannot fail"); + size = req.actual_size(); assert(r->top() <= r->end(), "Allocation cannot span end of region"); // actual_size() will be set to size below. assert((result == nullptr) || (size % CardTable::card_size_in_words() == 0), "PLAB size must be multiple of card size"); assert((result == nullptr) || (((uintptr_t) result) % CardTable::card_size_in_words() == 0), "PLAB start must align with card boundary"); - - if (result != nullptr && free > usable_free) { + if (free > usable_free) { // Account for the alignment padding size_t padding = (free - usable_free) * HeapWordSize; increase_used(padding); @@ -290,6 +289,10 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah } if (size >= req.min_size()) { result = r->allocate(size, req); + if (result != nullptr) { + // Record actual allocation size + req.set_actual_size(size); + } assert (result != NULL, "Allocation must succeed: free " SIZE_FORMAT ", actual " SIZE_FORMAT, free, size); } else { log_info(gc, ergo)("Failed to shrink TLAB or GCLAB request (" SIZE_FORMAT ") in region " SIZE_FORMAT " to " SIZE_FORMAT @@ -297,6 +300,7 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah } } } else if (req.is_lab_alloc() && req.type() == ShenandoahAllocRequest::_alloc_plab) { + // inelastic PLAB size_t free = r->free(); size_t usable_free = (free / CardTable::card_size()) << CardTable::card_shift(); free /= HeapWordSize; @@ -312,6 +316,7 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah assert(size % CardTable::card_size_in_words() == 0, "PLAB size must be multiple of remembered set card size"); if (size <= usable_free) { result = r->allocate_aligned(size, req, CardTable::card_size()); + size = req.actual_size(); assert(result != nullptr, "Allocation cannot fail"); assert(r->top() <= r->end(), "Allocation cannot span end of region"); assert(req.actual_size() % CardTable::card_size_in_words() == 0, "PLAB start must align with card boundary"); @@ -328,12 +333,13 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah } } else { result = r->allocate(size, req); + if (result != nullptr) { + // Record actual allocation size + req.set_actual_size(size); + } } if (result != NULL) { - // Record actual allocation size - req.set_actual_size(size); - // Allocation successful, bump stats: if (req.is_mutator_alloc()) { // Mutator allocations always pull from young gen. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp index 101fa2e76e6df..824dd22e7082f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp @@ -348,7 +348,7 @@ class ShenandoahHeapRegion { } // Allocation (return NULL if full) - inline HeapWord* allocate_aligned(size_t word_size, ShenandoahAllocRequest req, size_t alignment_in_words); + inline HeapWord* allocate_aligned(size_t word_size, ShenandoahAllocRequest &req, size_t alignment_in_words); // Allocation (return NULL if full) inline HeapWord* allocate(size_t word_size, ShenandoahAllocRequest req); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp index a37e14a0596f5..7e7539e5c742d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp @@ -35,43 +35,66 @@ // so that returned object is aligned on an address that is a multiple of alignment_in_words. Requested // size is in words. It is assumed that this->is_old(). A pad object is allocated, filled, and registered // if necessary to assure the new allocation is properly aligned. -HeapWord* ShenandoahHeapRegion::allocate_aligned(size_t size, ShenandoahAllocRequest req, size_t alignment_in_bytes) { +HeapWord* ShenandoahHeapRegion::allocate_aligned(size_t size, ShenandoahAllocRequest &req, size_t alignment_in_bytes) { shenandoah_assert_heaplocked_or_safepoint(); + assert(req.is_lab_alloc(), "allocate_aligned() only applies to LAB allocations"); assert(is_object_aligned(size), "alloc size breaks alignment: " SIZE_FORMAT, size); assert(is_old(), "aligned allocations are only taken from OLD regions to support PLABs"); - HeapWord* obj = top(); - uintptr_t addr_as_int = (uintptr_t) obj; + HeapWord* orig_top = top(); + size_t addr_as_int = (uintptr_t) orig_top; - size_t unalignment_bytes = addr_as_int % alignment_in_bytes; - assert(unalignment_bytes % HeapWordSize == 0, "top should be multiple of HeapWordSize"); + // unalignment_bytes is the amount by which current top() exceeds the desired alignment point. We subtract this amount + // from alignment_in_bytes to determine padding required to next alignment point. - size_t pad_words = 0; - if (unalignment_bytes > 0) { - pad_words = (alignment_in_bytes - unalignment_bytes) / HeapWordSize; + // top is HeapWord-aligned so unalignment_bytes is a multiple of HeapWordSize + size_t unalignment_bytes = addr_as_int % alignment_in_bytes; + size_t unalignment_words = unalignment_bytes / HeapWordSize; + + size_t pad_words; + HeapWord* aligned_obj; + if (unalignment_words > 0) { + pad_words = (alignment_in_bytes / HeapWordSize) - unalignment_words; + if (pad_words < ShenandoahHeap::min_fill_size()) { + pad_words += (alignment_in_bytes / HeapWordSize); + } + aligned_obj = orig_top + pad_words; + } else { + pad_words = 0; + aligned_obj = orig_top; } - if ((pad_words > 0) && (pad_words < ShenandoahHeap::min_fill_size())) { - pad_words += alignment_in_bytes / HeapWordSize; + + if (pointer_delta(end(), aligned_obj) < size) { + size = pointer_delta(end(), aligned_obj); + // Force size to align on multiple of alignment_in_bytes + size_t byte_size = size * HeapWordSize; + size_t excess_bytes = byte_size % alignment_in_bytes; + // Note: excess_bytes is a multiple of HeapWordSize because it is the difference of HeapWord-aligned end + // and proposed HeapWord-aligned object address. + if (excess_bytes > 0) { + size -= excess_bytes / HeapWordSize; + } } - if (pointer_delta(end(), obj + pad_words) >= size) { + + // Both originally requested size and adjusted size must be properly aligned + assert ((size * HeapWordSize) % alignment_in_bytes == 0, "Size must be multiple of alignment constraint"); + if (size >= req.min_size()) { + // Even if req.min_size() is not a multiple of card size, we know that size is. if (pad_words > 0) { - ShenandoahHeap::fill_with_object(obj, pad_words); - // register the filled pad object - ShenandoahHeap::heap()->card_scan()->register_object(obj); - obj += pad_words; + ShenandoahHeap::fill_with_object(orig_top, pad_words); + ShenandoahHeap::heap()->card_scan()->register_object(orig_top); } - // We don't need to register the PLAB. Its content will be registered as objects are allocated within it and/or - // when the PLAB is retired. make_regular_allocation(req.affiliation()); adjust_alloc_metadata(req.type(), size); - HeapWord* new_top = obj + size; + HeapWord* new_top = aligned_obj + size; + assert(new_top <= end(), "PLAB cannot span end of heap region"); set_top(new_top); + req.set_actual_size(size); assert(is_object_aligned(new_top), "new top breaks alignment: " PTR_FORMAT, p2i(new_top)); - assert(is_aligned(obj, alignment_in_bytes), "obj is not aligned: " PTR_FORMAT, p2i(obj)); - - return obj; + assert(is_aligned(aligned_obj, alignment_in_bytes), "obj is not aligned: " PTR_FORMAT, p2i(aligned_obj)); + return aligned_obj; } else { return NULL; } From aca12fcb017524fc3107aa65e8f1566fc2e044fa Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Wed, 11 Jan 2023 16:42:04 +0000 Subject: [PATCH 173/254] Fix verification of remembered set at mark start Reviewed-by: wkemper, ysr --- src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 119588089363f..2115e128010b1 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -3151,6 +3151,7 @@ void ShenandoahHeap::verify_rem_set_at_mark() { while (iterator.has_next()) { ShenandoahHeapRegion* r = iterator.next(); + HeapWord* tams = ctx? ctx->top_at_mark_start(r): nullptr; if (r == nullptr) break; if (r->is_old() && r->is_active()) { @@ -3185,13 +3186,13 @@ void ShenandoahHeap::verify_rem_set_at_mark() { // else, object's start is marked dirty and obj is not an objArray, so any interesting pointers are covered if (!scanner->verify_registration(obj_addr, ctx)) { ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, obj_addr, NULL, - "Verify init-mark remembered set violation", "object not properly registered", __FILE__, __LINE__); + "Verify init-mark remembered set violation", "object not properly registered", __FILE__, __LINE__); } obj_addr += obj->size(); } else { // This object is not live so we don't verify dirty cards contained therein - assert(ctx->top_at_mark_start(r) == top, "Expect tams == top at start of mark."); - obj_addr = ctx->get_next_marked_addr(obj_addr, top); + assert(tams != nullptr, "If object is not live, ctx and tams should be non-null"); + obj_addr = ctx->get_next_marked_addr(obj_addr, tams); } } } // else, we ignore humongous continuation region From 0be422bf31db81b640ab0911a327a65e5c56381a Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Thu, 12 Jan 2023 21:45:42 +0000 Subject: [PATCH 174/254] Broaden plab region search Reviewed-by: wkemper --- .../gc/shenandoah/shenandoahConcurrentGC.cpp | 4 --- .../gc/shenandoah/shenandoahControlThread.cpp | 25 +++++++------ .../gc/shenandoah/shenandoahControlThread.hpp | 1 - .../gc/shenandoah/shenandoahDegeneratedGC.cpp | 6 +--- .../share/gc/shenandoah/shenandoahFreeSet.cpp | 36 ++++++++++++++++++- .../share/gc/shenandoah/shenandoahFreeSet.hpp | 1 + .../share/gc/shenandoah/shenandoahFullGC.cpp | 7 +--- .../gc/shenandoah/shenandoahGeneration.cpp | 13 ++++--- .../gc/shenandoah/shenandoahGeneration.hpp | 2 +- .../share/gc/shenandoah/shenandoahHeap.cpp | 10 ++++++ .../share/gc/shenandoah/shenandoahHeap.hpp | 3 ++ 11 files changed, 74 insertions(+), 34 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index 1e04ad0e15b87..9176b625d21a5 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -243,11 +243,7 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) { heap->reset_old_evac_expended(); heap->set_promoted_reserve(0); } - log_info(gc, ergo)("At end of Concurrent GC, old_available: " SIZE_FORMAT "%s, young_available: " SIZE_FORMAT "%s", - byte_size_in_proper_unit(old_available), proper_unit_for_byte_size(old_available), - byte_size_in_proper_unit(young_available), proper_unit_for_byte_size(young_available)); } - return true; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index 223f2afd28a1b..df70d4ffa9e7f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -474,17 +474,24 @@ void ShenandoahControlThread::service_concurrent_normal_cycle( default: ShouldNotReachHere(); } - log_heap_status(heap); + const char* msg; + if (heap->cancelled_gc()) { + msg = (generation == YOUNG)? "At end of Interrupted Concurrent Young GC": "At end of Interrupted Concurrent Bootstrap GC"; + } else { + msg = (generation == YOUNG)? "At end of Concurrent Young GC": "At end of Concurrent Bootstrap GC"; + } + heap->log_heap_status(msg); } void ShenandoahControlThread::service_concurrent_old_cycle(const ShenandoahHeap* heap, GCCause::Cause &cause) { ShenandoahOldGeneration* old_generation = heap->old_generation(); ShenandoahYoungGeneration* young_generation = heap->young_generation(); + ShenandoahOldGeneration::State original_state = old_generation->state(); TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); - switch (old_generation->state()) { + switch (original_state) { case ShenandoahOldGeneration::IDLE: { assert(!heap->is_concurrent_old_mark_in_progress(), "Old already in progress."); assert(old_generation->task_queues()->is_empty(), "Old mark queues should be empty."); @@ -537,6 +544,11 @@ void ShenandoahControlThread::service_concurrent_old_cycle(const ShenandoahHeap* bool marking_complete = resume_concurrent_old_cycle(old_generation, cause); if (marking_complete) { assert(old_generation->state() != ShenandoahOldGeneration::MARKING, "Should not still be marking."); + if (original_state == ShenandoahOldGeneration::MARKING) { + heap->log_heap_status("At end of Concurrent Old Marking finishing increment"); + } + } else if (original_state == ShenandoahOldGeneration::MARKING) { + heap->log_heap_status("At end of Concurrent Old Marking increment"); } break; } @@ -580,15 +592,6 @@ bool ShenandoahControlThread::resume_concurrent_old_cycle(ShenandoahGeneration* return true; } -void ShenandoahControlThread::log_heap_status(const ShenandoahHeap* heap) { - if (heap->mode()->is_generational()) { - heap->young_generation()->log_status(); - heap->old_generation()->log_status(); - } else { - heap->global_generation()->log_status(); - } -} - bool ShenandoahControlThread::check_soft_max_changed() const { ShenandoahHeap* heap = ShenandoahHeap::heap(); size_t new_soft_max = Atomic::load(&SoftMaxHeapSize); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp index 3af9112ffe86f..6f1974574e5db 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp @@ -190,7 +190,6 @@ class ShenandoahControlThread: public ConcurrentGCThread { void service_concurrent_cycle(const ShenandoahHeap* heap, ShenandoahGeneration* generation, GCCause::Cause &cause, bool do_old_gc_bootstrap); - void log_heap_status(const ShenandoahHeap* heap); }; #endif // SHARE_GC_SHENANDOAH_SHENANDOAHCONTROLTHREAD_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp index 91368b978985a..fdaca691770a8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp @@ -56,11 +56,7 @@ bool ShenandoahDegenGC::collect(GCCause::Cause cause) { vmop_degenerated(); ShenandoahHeap* heap = ShenandoahHeap::heap(); if (heap->mode()->is_generational()) { - size_t old_available = heap->old_generation()->available(); - size_t young_available = heap->young_generation()->available(); - log_info(gc, ergo)("At end of Degenerated GC, old_available: " SIZE_FORMAT "%s, young_available: " SIZE_FORMAT "%s", - byte_size_in_proper_unit(old_available), proper_unit_for_byte_size(old_available), - byte_size_in_proper_unit(young_available), proper_unit_for_byte_size(young_available)); + heap->log_heap_status("At end of Degenerated GC"); } return true; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index 3f6efc8520014..f5f063078721e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -65,6 +65,33 @@ bool ShenandoahFreeSet::is_collector_free(size_t idx) const { return _collector_free_bitmap.at(idx); } +// This is a temporary solution to work around a shortcoming with the existing free set implementation. +// TODO: +// Remove this function after restructing FreeSet representation. A problem in the existing implementation is that old-gen +// regions are not considered to reside within the is_collector_free range. +// +HeapWord* ShenandoahFreeSet::allocate_with_old_affiliation(ShenandoahAllocRequest& req, bool& in_new_region) { + ShenandoahRegionAffiliation affiliation = ShenandoahRegionAffiliation::OLD_GENERATION; + + size_t rightmost = MAX2(_collector_rightmost, _mutator_rightmost); + size_t leftmost = MIN2(_collector_leftmost, _mutator_leftmost); + + for (size_t c = rightmost + 1; c > leftmost; c--) { + // size_t is unsigned, need to dodge underflow when _leftmost = 0 + size_t idx = c - 1; + ShenandoahHeapRegion* r = _heap->get_region(idx); + if (r->affiliation() == affiliation && !r->is_humongous()) { + if (!r->is_cset() && !has_no_alloc_capacity(r)) { + HeapWord* result = try_allocate_in(r, req, in_new_region); + if (result != NULL) { + return result; + } + } + } + } + return nullptr; +} + HeapWord* ShenandoahFreeSet::allocate_with_affiliation(ShenandoahRegionAffiliation affiliation, ShenandoahAllocRequest& req, bool& in_new_region) { for (size_t c = _collector_rightmost + 1; c > _collector_leftmost; c--) { // size_t is unsigned, need to dodge underflow when _leftmost = 0 @@ -145,7 +172,14 @@ HeapWord* ShenandoahFreeSet::allocate_single(ShenandoahAllocRequest& req, bool& case ShenandoahAllocRequest::_alloc_shared_gc: { // First try to fit into a region that is already in use in the same generation. - HeapWord* result = allocate_with_affiliation(req.affiliation(), req, in_new_region); + HeapWord* result; + if (req.affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) { + // TODO: this is a work around to address a deficiency in FreeSet representation. A better solution fixes + // the FreeSet implementation to deal more efficiently with old-gen regions as being in the "collector free set" + result = allocate_with_old_affiliation(req, in_new_region); + } else { + result = allocate_with_affiliation(req.affiliation(), req, in_new_region); + } if (result != NULL) { return result; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp index 153c759cc7a93..da944e7fa44d2 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp @@ -50,6 +50,7 @@ class ShenandoahFreeSet : public CHeapObj { HeapWord* try_allocate_in(ShenandoahHeapRegion* region, ShenandoahAllocRequest& req, bool& in_new_region); HeapWord* allocate_with_affiliation(ShenandoahRegionAffiliation affiliation, ShenandoahAllocRequest& req, bool& in_new_region); + HeapWord* allocate_with_old_affiliation(ShenandoahAllocRequest& req, bool& in_new_region); // While holding the heap lock, allocate memory for a single object which is to be entirely contained // within a single HeapRegion as characterized by req. The req.size() value is known to be less than or diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index 3522415b97780..5c540b6609efc 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -174,11 +174,7 @@ void ShenandoahFullGC::op_full(GCCause::Cause cause) { metrics.snap_after(); if (heap->mode()->is_generational()) { - size_t old_available = heap->old_generation()->available(); - size_t young_available = heap->young_generation()->available(); - log_info(gc, ergo)("At end of Full GC, old_available: " SIZE_FORMAT "%s, young_available: " SIZE_FORMAT "%s", - byte_size_in_proper_unit(old_available), proper_unit_for_byte_size(old_available), - byte_size_in_proper_unit(young_available), proper_unit_for_byte_size(young_available)); + heap->log_heap_status("At end of Full GC"); } if (metrics.is_good_progress()) { ShenandoahHeap::heap()->notify_gc_progress(); @@ -746,7 +742,6 @@ void ShenandoahPrepareForCompactionTask::work(uint worker_id) { if (from_region->has_live()) { _heap->marked_object_iterate(from_region, &cl); } - // Compacted the region to somewhere else? From-region is empty then. if (!cl.is_compact_same_region()) { empty_regions.append(from_region); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index 94d78514e7ece..26de2e462b3e4 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -142,7 +142,7 @@ void ShenandoahGeneration::increase_allocated(size_t bytes) { Atomic::add(&_bytes_allocated_since_gc_start, bytes, memory_order_relaxed); } -void ShenandoahGeneration::log_status() const { +void ShenandoahGeneration::log_status(const char *msg) const { typedef LogTarget(Info, gc, ergo) LogGcInfo; if (!LogGcInfo::is_enabled()) { @@ -156,14 +156,17 @@ void ShenandoahGeneration::log_status() const { size_t v_soft_max_capacity = soft_max_capacity(); size_t v_max_capacity = max_capacity(); size_t v_available = available(); - LogGcInfo::print("%s generation used: " SIZE_FORMAT "%s, used regions: " SIZE_FORMAT "%s, " - "soft capacity: " SIZE_FORMAT "%s, max capacity: " SIZE_FORMAT " %s, available: " SIZE_FORMAT " %s", - name(), + size_t v_adjusted_avail = adjusted_available(); + LogGcInfo::print("%s: %s generation used: " SIZE_FORMAT "%s, used regions: " SIZE_FORMAT "%s, " + "soft capacity: " SIZE_FORMAT "%s, max capacity: " SIZE_FORMAT "%s, available: " SIZE_FORMAT "%s, " + "adjusted available: " SIZE_FORMAT "%s", + msg, name(), byte_size_in_proper_unit(v_used), proper_unit_for_byte_size(v_used), byte_size_in_proper_unit(v_used_regions), proper_unit_for_byte_size(v_used_regions), byte_size_in_proper_unit(v_soft_max_capacity), proper_unit_for_byte_size(v_soft_max_capacity), byte_size_in_proper_unit(v_max_capacity), proper_unit_for_byte_size(v_max_capacity), - byte_size_in_proper_unit(v_available), proper_unit_for_byte_size(v_available)); + byte_size_in_proper_unit(v_available), proper_unit_for_byte_size(v_available), + byte_size_in_proper_unit(v_adjusted_avail), proper_unit_for_byte_size(v_adjusted_avail)); } void ShenandoahGeneration::reset_mark_bitmap() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index c5134853b0796..7a63cd0d3d060 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -125,7 +125,7 @@ class ShenandoahGeneration : public CHeapObj { _soft_max_capacity = soft_max_capacity; } - void log_status() const; + void log_status(const char* msg) const; // Used directly by FullGC void reset_mark_bitmap(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 2115e128010b1..7a18e0fb0064b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -3307,3 +3307,13 @@ ShenandoahGeneration* ShenandoahHeap::generation_for(ShenandoahRegionAffiliation ShouldNotReachHere(); return nullptr; } + +void ShenandoahHeap::log_heap_status(const char* msg) const { + if (mode()->is_generational()) { + young_generation()->log_status(msg); + old_generation()->log_status(msg); + } else { + global_generation()->log_status(msg); + } +} + diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index 751b57f72025e..f2c042d00a6ab 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -855,6 +855,9 @@ class ShenandoahHeap : public CollectedHeap { static inline uint get_object_age(oop obj); void transfer_old_pointers_from_satb(); + + void log_heap_status(const char *msg) const; + private: void trash_cset_regions(); From 0e15cb6dfcd97a816f4213cd38ffdd5f402536b9 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Fri, 13 Jan 2023 17:56:18 +0000 Subject: [PATCH 175/254] Fix fullgc assertion Reviewed-by: ysr, wkemper --- src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp | 7 +++++++ .../share/gc/shenandoah/shenandoahHeapRegion.cpp | 10 ++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index 5c540b6609efc..767a08e7a5a2b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -175,6 +175,13 @@ void ShenandoahFullGC::op_full(GCCause::Cause cause) { metrics.snap_after(); if (heap->mode()->is_generational()) { heap->log_heap_status("At end of Full GC"); + + // Since we allow temporary violation of these constraints during Full GC, we want to enforce that the assertions are + // made valid by the time Full GC completes. + assert(heap->old_generation()->used_regions_size() <= heap->old_generation()->adjusted_capacity(), + "Old generation affiliated regions must be less than capacity"); + assert(heap->young_generation()->used_regions_size() <= heap->young_generation()->adjusted_capacity(), + "Young generation affiliated regions must be less than capacity"); } if (metrics.is_good_progress()) { ShenandoahHeap::heap()->notify_gc_progress(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index 4521de649951e..4bb3c16d9c98c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -982,12 +982,18 @@ void ShenandoahHeapRegion::set_affiliation(ShenandoahRegionAffiliation new_affil case YOUNG_GENERATION: reset_age(); regions = heap->young_generation()->increment_affiliated_region_count(); - assert(regions * ShenandoahHeapRegion::region_size_bytes() <= heap->young_generation()->adjusted_capacity(), + // During Full GC, we allow temporary violation of this requirement. We enforce that this condition is + // restored upon completion of Full GC. + assert(heap->is_full_gc_in_progress() || + (regions * ShenandoahHeapRegion::region_size_bytes() <= heap->young_generation()->adjusted_capacity()), "Number of young regions cannot exceed adjusted capacity"); break; case OLD_GENERATION: regions = heap->old_generation()->increment_affiliated_region_count(); - assert(regions * ShenandoahHeapRegion::region_size_bytes() <= heap->old_generation()->adjusted_capacity(), + // During Full GC, we allow temporary violation of this requirement. We enforce that this condition is + // restored upon completion of Full GC. + assert(heap->is_full_gc_in_progress() || + (regions * ShenandoahHeapRegion::region_size_bytes() <= heap->old_generation()->adjusted_capacity()), "Number of old regions cannot exceed adjusted capacity"); break; default: From ec3e5ef150e1ff7b1e35a450653d8bf0bb1ee6c9 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Fri, 13 Jan 2023 18:25:44 +0000 Subject: [PATCH 176/254] Use whole number of regions when resizing generations Reviewed-by: kdnilsen --- .../gc/shenandoah/shenandoahMmuTracker.cpp | 105 +++++++++--------- .../gc/shenandoah/shenandoahMmuTracker.hpp | 27 +++-- 2 files changed, 71 insertions(+), 61 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp index ea87186af1c8b..2c724337b4736 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp @@ -119,8 +119,8 @@ void ShenandoahMmuTracker::initialize() { ShenandoahGenerationSizer::ShenandoahGenerationSizer(ShenandoahMmuTracker* mmu_tracker) : _sizer_kind(SizerDefaults), _use_adaptive_sizing(true), - _min_desired_young_size(0), - _max_desired_young_size(0), + _min_desired_young_regions(0), + _max_desired_young_regions(0), _resize_increment(double(YoungGenerationSizeIncrement) / 100.0), _mmu_tracker(mmu_tracker) { @@ -144,64 +144,62 @@ ShenandoahGenerationSizer::ShenandoahGenerationSizer(ShenandoahMmuTracker* mmu_t } if (FLAG_IS_CMDLINE(NewSize)) { - _min_desired_young_size = MAX2(NewSize, ShenandoahHeapRegion::region_size_bytes()); + _min_desired_young_regions = MAX2(uint(NewSize / ShenandoahHeapRegion::region_size_bytes()), 1U); if (FLAG_IS_CMDLINE(MaxNewSize)) { - _max_desired_young_size = MAX2(MaxNewSize, ShenandoahHeapRegion::region_size_bytes()); + _max_desired_young_regions = MAX2(uint(MaxNewSize / ShenandoahHeapRegion::region_size_bytes()), 1U); _sizer_kind = SizerMaxAndNewSize; - _use_adaptive_sizing = _min_desired_young_size != _max_desired_young_size; + _use_adaptive_sizing = _min_desired_young_regions != _max_desired_young_regions; } else { _sizer_kind = SizerNewSizeOnly; } } else if (FLAG_IS_CMDLINE(MaxNewSize)) { - _max_desired_young_size = MAX2(MaxNewSize, ShenandoahHeapRegion::region_size_bytes()); + _max_desired_young_regions = MAX2(uint(MaxNewSize / ShenandoahHeapRegion::region_size_bytes()), 1U); _sizer_kind = SizerMaxNewSizeOnly; } } -size_t ShenandoahGenerationSizer::calculate_min_size(size_t heap_size) { - size_t default_value = (heap_size * ShenandoahMinYoungPercentage) / 100; - default_value &= ~ShenandoahHeapRegion::region_size_bytes_mask(); - return MAX2(ShenandoahHeapRegion::region_size_bytes(), default_value); +size_t ShenandoahGenerationSizer::calculate_min_young_regions(size_t heap_region_count) { + size_t min_young_regions = (heap_region_count * ShenandoahMinYoungPercentage) / 100; + return MAX2(uint(min_young_regions), 1U); } -size_t ShenandoahGenerationSizer::calculate_max_size(size_t heap_size) { - size_t default_value = (heap_size * ShenandoahMaxYoungPercentage) / 100; - default_value &= ~ShenandoahHeapRegion::region_size_bytes_mask(); - return MAX2(ShenandoahHeapRegion::region_size_bytes(), default_value); +size_t ShenandoahGenerationSizer::calculate_max_young_regions(size_t heap_region_count) { + size_t max_young_regions = (heap_region_count * ShenandoahMaxYoungPercentage) / 100; + return MAX2(uint(max_young_regions), 1U); } -void ShenandoahGenerationSizer::recalculate_min_max_young_length(size_t heap_size) { - assert(heap_size > 0, "Heap must be initialized"); +void ShenandoahGenerationSizer::recalculate_min_max_young_length(size_t heap_region_count) { + assert(heap_region_count > 0, "Heap must be initialized"); switch (_sizer_kind) { case SizerDefaults: - _min_desired_young_size = calculate_min_size(heap_size); - _max_desired_young_size = calculate_max_size(heap_size); + _min_desired_young_regions = calculate_min_young_regions(heap_region_count); + _max_desired_young_regions = calculate_max_young_regions(heap_region_count); break; case SizerNewSizeOnly: - _max_desired_young_size = calculate_max_size(heap_size); - _max_desired_young_size = MAX2(_min_desired_young_size, _max_desired_young_size); + _max_desired_young_regions = calculate_max_young_regions(heap_region_count); + _max_desired_young_regions = MAX2(_min_desired_young_regions, _max_desired_young_regions); break; case SizerMaxNewSizeOnly: - _min_desired_young_size = calculate_min_size(heap_size); - _min_desired_young_size = MIN2(_min_desired_young_size, _max_desired_young_size); + _min_desired_young_regions = calculate_min_young_regions(heap_region_count); + _min_desired_young_regions = MIN2(_min_desired_young_regions, _max_desired_young_regions); break; case SizerMaxAndNewSize: // Do nothing. Values set on the command line, don't update them at runtime. break; case SizerNewRatio: - _min_desired_young_size = MAX2((heap_size / (NewRatio + 1)), ShenandoahHeapRegion::region_size_bytes()); - _max_desired_young_size = _min_desired_young_size; + _min_desired_young_regions = MAX2(uint(heap_region_count / (NewRatio + 1)), 1U); + _max_desired_young_regions = _min_desired_young_regions; break; default: ShouldNotReachHere(); } - assert(_min_desired_young_size <= _max_desired_young_size, "Invalid min/max young gen size values"); + assert(_min_desired_young_regions <= _max_desired_young_regions, "Invalid min/max young gen size values"); } void ShenandoahGenerationSizer::heap_size_changed(size_t heap_size) { - recalculate_min_max_young_length(heap_size); + recalculate_min_max_young_length(heap_size / ShenandoahHeapRegion::region_size_bytes()); } bool ShenandoahGenerationSizer::adjust_generation_sizes() const { @@ -261,52 +259,59 @@ bool ShenandoahGenerationSizer::transfer_capacity(ShenandoahGeneration* from, Sh } size_t regions_to_transfer = MAX2(1u, uint(double(available_regions) * _resize_increment)); - size_t bytes_to_transfer = regions_to_transfer * ShenandoahHeapRegion::region_size_bytes(); if (from->generation_mode() == YOUNG) { - bytes_to_transfer = adjust_transfer_from_young(from, bytes_to_transfer); + regions_to_transfer = adjust_transfer_from_young(from, regions_to_transfer); } else { - bytes_to_transfer = adjust_transfer_to_young(to, bytes_to_transfer); + regions_to_transfer = adjust_transfer_to_young(to, regions_to_transfer); } - if (bytes_to_transfer == 0) { - log_debug(gc)("No capacity available to transfer from: %s (" SIZE_FORMAT "%s) to: %s (" SIZE_FORMAT "%s)", + if (regions_to_transfer == 0) { + log_info(gc)("No capacity available to transfer from: %s (" SIZE_FORMAT "%s) to: %s (" SIZE_FORMAT "%s)", from->name(), byte_size_in_proper_unit(from->max_capacity()), proper_unit_for_byte_size(from->max_capacity()), to->name(), byte_size_in_proper_unit(to->max_capacity()), proper_unit_for_byte_size(to->max_capacity())); return false; } - assert(bytes_to_transfer <= regions_to_transfer * ShenandoahHeapRegion::region_size_bytes(), "Cannot transfer more than available in free regions."); - log_info(gc)("Transfer " SIZE_FORMAT "%s from %s to %s", byte_size_in_proper_unit(bytes_to_transfer), - proper_unit_for_byte_size(bytes_to_transfer), from->name(), to->name()); - from->decrease_capacity(bytes_to_transfer); - to->increase_capacity(bytes_to_transfer); + log_info(gc)("Transfer " SIZE_FORMAT " region(s) from %s to %s", regions_to_transfer, from->name(), to->name()); + from->decrease_capacity(regions_to_transfer * ShenandoahHeapRegion::region_size_bytes()); + to->increase_capacity(regions_to_transfer * ShenandoahHeapRegion::region_size_bytes()); return true; } -size_t ShenandoahGenerationSizer::adjust_transfer_from_young(ShenandoahGeneration* from, size_t bytes_to_transfer) const { +size_t ShenandoahGenerationSizer::adjust_transfer_from_young(ShenandoahGeneration* from, size_t regions_to_transfer) const { assert(from->generation_mode() == YOUNG, "Expect to transfer from young"); - size_t new_young_size = from->max_capacity() - bytes_to_transfer; - size_t minimum_size = min_young_size(); + size_t young_capacity_regions = from->max_capacity() / ShenandoahHeapRegion::region_size_bytes(); + size_t new_young_regions = young_capacity_regions - regions_to_transfer; + size_t minimum_young_regions = min_young_regions(); // Check that we are not going to violate the minimum size constraint. - if (new_young_size < minimum_size) { - assert(minimum_size <= from->max_capacity(), "Young is under minimum capacity."); + if (new_young_regions < minimum_young_regions) { + assert(minimum_young_regions <= young_capacity_regions, "Young is under minimum capacity."); // If the transfer violates the minimum size and there is still some capacity to transfer, // adjust the transfer to take the size to the minimum. Note that this may be zero. - bytes_to_transfer = align_down(from->max_capacity() - minimum_size, ShenandoahHeapRegion::region_size_bytes()); + regions_to_transfer = young_capacity_regions - minimum_young_regions; } - return bytes_to_transfer; + return regions_to_transfer; } -size_t ShenandoahGenerationSizer::adjust_transfer_to_young(ShenandoahGeneration* to, size_t bytes_to_transfer) const { +size_t ShenandoahGenerationSizer::adjust_transfer_to_young(ShenandoahGeneration* to, size_t regions_to_transfer) const { assert(to->generation_mode() == YOUNG, "Can only transfer between young and old."); - size_t new_young_size = to->max_capacity() + bytes_to_transfer; - size_t maximum_size = max_young_size(); + size_t young_capacity_regions = to->max_capacity() / ShenandoahHeapRegion::region_size_bytes(); + size_t new_young_regions = young_capacity_regions + regions_to_transfer; + size_t maximum_young_regions = max_young_regions(); // Check that we are not going to violate the maximum size constraint. - if (new_young_size > maximum_size) { - assert(maximum_size >= to->max_capacity(), "Young is over maximum capacity"); + if (new_young_regions > maximum_young_regions) { + assert(maximum_young_regions >= young_capacity_regions, "Young is over maximum capacity"); // If the transfer violates the maximum size and there is still some capacity to transfer, // adjust the transfer to take the size to the maximum. Note that this may be zero. - bytes_to_transfer = align_down(maximum_size - to->max_capacity(), ShenandoahHeapRegion::region_size_bytes()); + regions_to_transfer = maximum_young_regions - young_capacity_regions; } - return bytes_to_transfer; + return regions_to_transfer; +} + +size_t ShenandoahGenerationSizer::min_young_size() const { + return min_young_regions() * ShenandoahHeapRegion::region_size_bytes(); +} + +size_t ShenandoahGenerationSizer::max_young_size() const { + return max_young_regions() * ShenandoahHeapRegion::region_size_bytes(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.hpp index f322bb0500f99..f166a74b32998 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.hpp @@ -101,23 +101,23 @@ class ShenandoahGenerationSizer { // true otherwise. bool _use_adaptive_sizing; - size_t _min_desired_young_size; - size_t _max_desired_young_size; + size_t _min_desired_young_regions; + size_t _max_desired_young_regions; double _resize_increment; ShenandoahMmuTracker* _mmu_tracker; - static size_t calculate_min_size(size_t heap_size); - static size_t calculate_max_size(size_t heap_size); + static size_t calculate_min_young_regions(size_t heap_region_count); + static size_t calculate_max_young_regions(size_t heap_region_count); // Update the given values for minimum and maximum young gen length in regions // given the number of heap regions depending on the kind of sizing algorithm. - void recalculate_min_max_young_length(size_t heap_size); + void recalculate_min_max_young_length(size_t heap_region_count); // These two methods are responsible for enforcing the minimum and maximum // constraints for the size of the generations. - size_t adjust_transfer_from_young(ShenandoahGeneration* from, size_t bytes_to_transfer) const; - size_t adjust_transfer_to_young(ShenandoahGeneration* to, size_t bytes_to_transfer) const; + size_t adjust_transfer_from_young(ShenandoahGeneration* from, size_t regions_to_transfer) const; + size_t adjust_transfer_to_young(ShenandoahGeneration* to, size_t regions_to_transfer) const; // This will attempt to transfer capacity from one generation to the other. It // returns true if a transfer is made, false otherwise. @@ -129,11 +129,16 @@ class ShenandoahGenerationSizer { // depending on the sizing algorithm. void heap_size_changed(size_t heap_size); - size_t min_young_size() const { - return _min_desired_young_size; + // Minimum size of young generation in bytes as multiple of region size. + size_t min_young_size() const; + size_t min_young_regions() const { + return _min_desired_young_regions; } - size_t max_young_size() const { - return _max_desired_young_size; + + // Maximum size of young generation in bytes as multiple of region size. + size_t max_young_size() const; + size_t max_young_regions() const { + return _max_desired_young_regions; } bool use_adaptive_sizing() const { From 2cab4e76083743f6eabf427bfc5ec8c4b3eaf081 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Wed, 18 Jan 2023 00:05:57 +0000 Subject: [PATCH 177/254] Do not reset learning cycles after resizing Reviewed-by: ysr, kdnilsen --- .../heuristics/shenandoahAdaptiveHeuristics.cpp | 12 ++++++++---- .../heuristics/shenandoahAdaptiveHeuristics.hpp | 11 +++++++++++ .../share/gc/shenandoah/shenandoahGeneration.cpp | 2 -- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp index 0a039b2c31e37..909879aea8ebd 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp @@ -55,6 +55,8 @@ const double ShenandoahAdaptiveHeuristics::HIGHEST_EXPECTED_AVAILABLE_AT_END = 0 const double ShenandoahAdaptiveHeuristics::MINIMUM_CONFIDENCE = 0.319; // 25% const double ShenandoahAdaptiveHeuristics::MAXIMUM_CONFIDENCE = 3.291; // 99.9% +const uint ShenandoahAdaptiveHeuristics::MINIMUM_RESIZE_INTERVAL = 10; + ShenandoahAdaptiveHeuristics::ShenandoahAdaptiveHeuristics(ShenandoahGeneration* generation) : ShenandoahHeuristics(generation), _margin_of_error_sd(ShenandoahAdaptiveInitialConfidence), @@ -235,6 +237,7 @@ void ShenandoahAdaptiveHeuristics::choose_collection_set_from_regiondata(Shenand void ShenandoahAdaptiveHeuristics::record_cycle_start() { ShenandoahHeuristics::record_cycle_start(); _allocation_rate.allocation_counter_reset(); + ++_cycles_since_last_resize; } void ShenandoahAdaptiveHeuristics::record_success_concurrent(bool abbreviated) { @@ -483,18 +486,19 @@ bool ShenandoahAdaptiveHeuristics::resize_and_evaluate() { return true; } - if (_gc_times_learned < ShenandoahLearningSteps) { - // We aren't going to attempt to resize our generation until we have 'learned' - // something about it. This provides a kind of cool down period after we've made - // a change, to help prevent thrashing. + if (_cycles_since_last_resize <= MINIMUM_RESIZE_INTERVAL) { + log_info(gc, ergo)("Not resizing %s for another " UINT32_FORMAT " cycles.", + _generation->name(), _cycles_since_last_resize); return true; } if (!heap->generation_sizer()->transfer_capacity(_generation)) { // We could not enlarge our generation, so we must start a gc cycle. + log_info(gc, ergo)("Could not increase size of %s, begin gc cycle.", _generation->name()); return true; } + log_info(gc)("Increased size of %s generation, re-evaluate trigger criteria", _generation->name()); return should_start_gc(); } diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp index 7c5f710c04e37..6681909ca7077 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp @@ -85,6 +85,13 @@ class ShenandoahAdaptiveHeuristics : public ShenandoahHeuristics { const static double LOWEST_EXPECTED_AVAILABLE_AT_END; const static double HIGHEST_EXPECTED_AVAILABLE_AT_END; + // At least this many cycles must execute before the heuristic will attempt + // to resize its generation. This is to prevent the heuristic from rapidly + // maxing out the generation size (which only forces the collector for the + // other generation to run more frequently, defeating the purpose of improving + // MMU). + const static uint MINIMUM_RESIZE_INTERVAL; + friend class ShenandoahAllocationRate; // Used to record the last trigger that signaled to start a GC. @@ -128,6 +135,10 @@ class ShenandoahAdaptiveHeuristics : public ShenandoahHeuristics { // establishes what is 'normal' for the application and is used as a // source of feedback to adjust trigger parameters. TruncatedSeq _available; + + // Do not attempt to resize the generation for this heuristic until this + // value is greater than MINIMUM_RESIZE_INTERVAL. + uint _cycles_since_last_resize; }; #endif // SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHADAPTIVEHEURISTICS_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index 26de2e462b3e4..b5024a3de4ba0 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -1005,7 +1005,6 @@ void ShenandoahGeneration::increase_capacity(size_t increment) { _max_capacity += increment; _soft_max_capacity += increment; _adjusted_capacity += increment; - heuristics()->reset_gc_learning(); } void ShenandoahGeneration::decrease_capacity(size_t decrement) { @@ -1014,7 +1013,6 @@ void ShenandoahGeneration::decrease_capacity(size_t decrement) { _max_capacity -= decrement; _soft_max_capacity -= decrement; _adjusted_capacity -= decrement; - heuristics()->reset_gc_learning(); } void ShenandoahGeneration::record_success_concurrent(bool abbreviated) { From 800c0c884631ae5b9c265c43dad8a49a17a8ec09 Mon Sep 17 00:00:00 2001 From: Stuart Monteith Date: Tue, 24 Jan 2023 00:39:39 +0000 Subject: [PATCH 178/254] 8298647: GenShen require heap size 2MB granularity Reviewed-by: wkemper --- src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index 4bb3c16d9c98c..d5c7b3cb1a1e0 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -23,6 +23,7 @@ */ #include "precompiled.hpp" +#include "gc/shared/cardTableRS.hpp" #include "gc/shared/space.inline.hpp" #include "gc/shared/tlab_globals.hpp" #include "gc/shenandoah/shenandoahCardTable.hpp" @@ -724,6 +725,11 @@ size_t ShenandoahHeapRegion::setup_sizes(size_t max_heap_size) { FLAG_SET_DEFAULT(ShenandoahMinRegionSize, MIN_REGION_SIZE); } + // Generational Shenandoah needs this alignment for card tables. + if (strcmp(ShenandoahGCMode, "generational") == 0) { + max_heap_size = align_up(max_heap_size , CardTableRS::ct_max_alignment_constraint()); + } + size_t region_size; if (FLAG_IS_DEFAULT(ShenandoahRegionSize)) { if (ShenandoahMinRegionSize > max_heap_size / MIN_NUM_REGIONS) { From 60861ba46a259f58c457c672038d59fef63499cc Mon Sep 17 00:00:00 2001 From: William Kemper Date: Fri, 27 Jan 2023 23:37:22 +0000 Subject: [PATCH 179/254] Revert unnecessary changes to unified logging Reviewed-by: kdnilsen, ysr --- src/hotspot/share/logging/logFileStreamOutput.cpp | 14 ++++++++++++++ src/hotspot/share/logging/logFileStreamOutput.hpp | 14 -------------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/hotspot/share/logging/logFileStreamOutput.cpp b/src/hotspot/share/logging/logFileStreamOutput.cpp index 22f71c995c4cc..715c13279f839 100644 --- a/src/hotspot/share/logging/logFileStreamOutput.cpp +++ b/src/hotspot/share/logging/logFileStreamOutput.cpp @@ -74,6 +74,20 @@ int LogFileStreamOutput::write_decorations(const LogDecorations& decorations) { return total_written; } +class FileLocker : public StackObj { +private: + FILE *_file; + +public: + FileLocker(FILE *file) : _file(file) { + os::flockfile(_file); + } + + ~FileLocker() { + os::funlockfile(_file); + } +}; + bool LogFileStreamOutput::flush() { bool result = true; if (fflush(_stream) != 0) { diff --git a/src/hotspot/share/logging/logFileStreamOutput.hpp b/src/hotspot/share/logging/logFileStreamOutput.hpp index 835601672b70a..cd1d34c086447 100644 --- a/src/hotspot/share/logging/logFileStreamOutput.hpp +++ b/src/hotspot/share/logging/logFileStreamOutput.hpp @@ -90,18 +90,4 @@ class LogStderrOutput : public LogFileStreamOutput { extern LogStderrOutput* StderrLog; extern LogStdoutOutput* StdoutLog; -class FileLocker : public StackObj { -private: - FILE *_file; - -public: - FileLocker(FILE *file) : _file(file) { - os::flockfile(_file); - } - - ~FileLocker() { - os::funlockfile(_file); - } -}; - #endif // SHARE_LOGGING_LOGFILESTREAMOUTPUT_HPP From 181326657c059ca497ea7c8610e0590dfb95e136 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Tue, 31 Jan 2023 20:56:50 +0000 Subject: [PATCH 180/254] Tune heuristic defaults and behavior for improved stability Reviewed-by: kdnilsen --- .../shenandoahAdaptiveHeuristics.cpp | 23 +++++++++++-------- .../heuristics/shenandoahHeuristics.cpp | 2 +- .../heuristics/shenandoahHeuristics.hpp | 4 +++- .../gc/shenandoah/shenandoahControlThread.cpp | 10 +++++--- .../share/gc/shenandoah/shenandoahFreeSet.cpp | 2 +- .../gc/shenandoah/shenandoah_globals.hpp | 4 ++-- 6 files changed, 27 insertions(+), 18 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp index 909879aea8ebd..3fd9042ab8830 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp @@ -61,7 +61,8 @@ ShenandoahAdaptiveHeuristics::ShenandoahAdaptiveHeuristics(ShenandoahGeneration* ShenandoahHeuristics(generation), _margin_of_error_sd(ShenandoahAdaptiveInitialConfidence), _spike_threshold_sd(ShenandoahAdaptiveInitialSpikeThreshold), - _last_trigger(OTHER) { } + _last_trigger(OTHER), + _available(Moving_Average_Samples, ShenandoahAdaptiveDecayFactor) { } ShenandoahAdaptiveHeuristics::~ShenandoahAdaptiveHeuristics() {} @@ -243,19 +244,21 @@ void ShenandoahAdaptiveHeuristics::record_cycle_start() { void ShenandoahAdaptiveHeuristics::record_success_concurrent(bool abbreviated) { ShenandoahHeuristics::record_success_concurrent(abbreviated); - size_t available = ShenandoahHeap::heap()->free_set()->available(); + size_t available = MIN2(_generation->available(), ShenandoahHeap::heap()->free_set()->available()); - _available.add(available); double z_score = 0.0; - if (_available.sd() > 0) { - z_score = (available - _available.avg()) / _available.sd(); + double available_sd = _available.sd(); + if (available_sd > 0) { + double available_avg = _available.avg(); + z_score = (double(available) - available_avg) / available_sd; + log_debug(gc, ergo)("%s Available: " SIZE_FORMAT " %sB, z-score=%.3f. Average available: %.1f %sB +/- %.1f %sB.", + _generation->name(), + byte_size_in_proper_unit(available), proper_unit_for_byte_size(available), z_score, + byte_size_in_proper_unit(available_avg), proper_unit_for_byte_size(available_avg), + byte_size_in_proper_unit(available_sd), proper_unit_for_byte_size(available_sd)); } - log_debug(gc, ergo)("Available: " SIZE_FORMAT " %sB, z-score=%.3f. Average available: %.1f %sB +/- %.1f %sB.", - byte_size_in_proper_unit(available), proper_unit_for_byte_size(available), - z_score, - byte_size_in_proper_unit(_available.avg()), proper_unit_for_byte_size(_available.avg()), - byte_size_in_proper_unit(_available.sd()), proper_unit_for_byte_size(_available.sd())); + _available.add(double(available)); // In the case when a concurrent GC cycle completes successfully but with an // unusually small amount of available memory we will adjust our trigger diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp index a26cb7919183c..e9e51c39bbe18 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp @@ -57,7 +57,7 @@ ShenandoahHeuristics::ShenandoahHeuristics(ShenandoahGeneration* generation) : _last_cycle_end(0), _gc_times_learned(0), _gc_time_penalties(0), - _gc_cycle_time_history(new TruncatedSeq(10, ShenandoahAdaptiveDecayFactor)), + _gc_cycle_time_history(new TruncatedSeq(Moving_Average_Samples, ShenandoahAdaptiveDecayFactor)), _live_memory_last_cycle(0), _live_memory_penultimate_cycle(0), _metaspace_oom() diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp index 380348fd966bf..73aa18c19c598 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp @@ -65,6 +65,8 @@ class ShenandoahHeuristics : public CHeapObj { static const intx Full_Penalty = 20; // how much to penalize average GC duration history on Full GC protected: + static const uint Moving_Average_Samples = 10; // Number of samples to store in moving averages + typedef struct { ShenandoahHeapRegion* _region; size_t _garbage; @@ -77,7 +79,7 @@ class ShenandoahHeuristics : public CHeapObj { // if (_generation->generation_mode() == OLD) _region_data represents // the results of most recently completed old-gen marking pass // if (_generation->generation_mode() == YOUNG) _region_data represents - // the resulits of most recently completed young-gen marking pass + // the results of most recently completed young-gen marking pass // // Note that there is some redundancy represented in _region_data because // each instance is an array large enough to hold all regions. However, diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index dbbe3e2df4f50..cdf329f99d031 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -475,10 +475,14 @@ void ShenandoahControlThread::service_concurrent_normal_cycle( ShouldNotReachHere(); } const char* msg; - if (heap->cancelled_gc()) { - msg = (generation == YOUNG)? "At end of Interrupted Concurrent Young GC": "At end of Interrupted Concurrent Bootstrap GC"; + if (heap->mode()->is_generational()) { + if (heap->cancelled_gc()) { + msg = (generation == YOUNG)? "At end of Interrupted Concurrent Young GC": "At end of Interrupted Concurrent Bootstrap GC"; + } else { + msg = (generation == YOUNG)? "At end of Concurrent Young GC": "At end of Concurrent Bootstrap GC"; + } } else { - msg = (generation == YOUNG)? "At end of Concurrent Young GC": "At end of Concurrent Bootstrap GC"; + msg = heap->cancelled_gc() ? "At end of cancelled GC" : "At end of GC"; } heap->log_heap_status(msg); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index f5f063078721e..5e31061151b4e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -329,7 +329,7 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah } assert (result != NULL, "Allocation must succeed: free " SIZE_FORMAT ", actual " SIZE_FORMAT, free, size); } else { - log_info(gc, ergo)("Failed to shrink TLAB or GCLAB request (" SIZE_FORMAT ") in region " SIZE_FORMAT " to " SIZE_FORMAT + log_trace(gc, ergo)("Failed to shrink TLAB or GCLAB request (" SIZE_FORMAT ") in region " SIZE_FORMAT " to " SIZE_FORMAT " because min_size() is " SIZE_FORMAT, req.size(), r->index(), size, req.min_size()); } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp index f92871240bce8..d02c949f5d6e6 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp @@ -147,7 +147,7 @@ "cases. In percents of (soft) max heap size.") \ range(0,100) \ \ - product(uintx, ShenandoahLearningSteps, 5, EXPERIMENTAL, \ + product(uintx, ShenandoahLearningSteps, 10, EXPERIMENTAL, \ "The number of cycles some heuristics take to collect in order " \ "to learn application and GC performance.") \ range(0,100) \ @@ -181,7 +181,7 @@ "the heuristic is to allocation spikes. Decreasing this number " \ "increases the sensitivity. ") \ \ - product(double, ShenandoahAdaptiveDecayFactor, 0.5, EXPERIMENTAL, \ + product(double, ShenandoahAdaptiveDecayFactor, 0.1, EXPERIMENTAL, \ "The decay factor (alpha) used for values in the weighted " \ "moving average of cycle time and allocation rate. " \ "Larger values give more weight to recent values.") \ From 8a67ed0d7c79eefc748f18361bd80911dfbac85f Mon Sep 17 00:00:00 2001 From: William Kemper Date: Tue, 31 Jan 2023 20:58:04 +0000 Subject: [PATCH 181/254] Combine bitmap clearing with region resetting closure Reviewed-by: kdnilsen, ysr --- .../gc/shenandoah/shenandoahGeneration.cpp | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index b5024a3de4ba0..96787bd07b545 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -39,12 +39,18 @@ class ShenandoahResetUpdateRegionStateClosure : public ShenandoahHeapRegionClosure { private: + ShenandoahHeap* _heap; ShenandoahMarkingContext* const _ctx; public: ShenandoahResetUpdateRegionStateClosure() : - _ctx(ShenandoahHeap::heap()->marking_context()) {} + _heap(ShenandoahHeap::heap()), + _ctx(_heap->marking_context()) {} + + void heap_region_do(ShenandoahHeapRegion* r) override { + if (_heap->is_bitmap_slice_committed(r)) { + _ctx->clear_bitmap(r); + } - void heap_region_do(ShenandoahHeapRegion* r) { if (r->is_active()) { // Reset live data and set TAMS optimistically. We would recheck these under the pause // anyway to capture any updates that happened since now. @@ -53,7 +59,7 @@ class ShenandoahResetUpdateRegionStateClosure : public ShenandoahHeapRegionClosu } } - bool is_thread_safe() { return true; } + bool is_thread_safe() override { return true; } }; class ShenandoahResetBitmapTask : public ShenandoahHeapRegionClosure { @@ -209,9 +215,10 @@ void ShenandoahGeneration::merge_write_table() { } void ShenandoahGeneration::prepare_gc() { - // Reset mark bitmap for this generation (typically young) - reset_mark_bitmap(); - // Capture Top At Mark Start for this generation (typically young) + // Invalidate the marking context + set_mark_incomplete(); + + // Capture Top At Mark Start for this generation (typically young) and reset mark bitmap. ShenandoahResetUpdateRegionStateClosure cl; parallel_heap_region_iterate(&cl); } From a2963b17370cdd2d45b8b19d49e23e3791424499 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Tue, 31 Jan 2023 22:47:22 +0000 Subject: [PATCH 182/254] Age objects during degeneration Reviewed-by: wkemper --- src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp | 4 +++- src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp | 5 ----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index cdf329f99d031..21914c94e3f7f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -267,7 +267,8 @@ void ShenandoahControlThread::run_service() { ShenandoahHeapLocker locker(heap->lock()); heap->free_set()->log_status(); } - + // In case this is a degenerated cycle, remember whether original cycle was aging. + bool was_aging_cycle = heap->is_aging_cycle(); heap->set_aging_cycle(false); { switch (_mode) { @@ -280,6 +281,7 @@ void ShenandoahControlThread::run_service() { break; } case stw_degenerated: { + heap->set_aging_cycle(was_aging_cycle); if (!service_stw_degenerated_cycle(cause, degen_point)) { // The degenerated GC was upgraded to a Full GC generation = GLOBAL; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp index fdaca691770a8..3b5ce4622fde4 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp @@ -74,11 +74,6 @@ void ShenandoahDegenGC::entry_degenerated() { ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::degen_gc, true /* log_heap_usage */); EventMark em("%s", msg); ShenandoahHeap* const heap = ShenandoahHeap::heap(); - - // In case degenerated GC preempted evacuation or update-refs, clear the aging cycle now. No harm in clearing it - // redundantly if it is already clear. We don't age during degenerated cycles. - heap->set_aging_cycle(false); - ShenandoahWorkerScope scope(heap->workers(), ShenandoahWorkerPolicy::calc_workers_for_stw_degenerated(), "stw degenerated gc"); From 998ea70c079ac4b2498178aade9d84eb6b2853ea Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Wed, 1 Feb 2023 16:55:20 +0000 Subject: [PATCH 183/254] Loan from old should align on region size Reviewed-by: ysr, wkemper --- src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index 96787bd07b545..ddfa10f6d4d4f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -729,13 +729,17 @@ void ShenandoahGeneration::adjust_evacuation_budgets(ShenandoahHeap* heap, Shena old_bytes_reserved_for_alloc_supplement += additional_regions_to_loan * region_size_bytes; working_old_available -= additional_regions_to_loan * region_size_bytes; } - size_t allocation_supplement = old_bytes_reserved_for_alloc_supplement; + size_t allocation_supplement = old_bytes_reserved_for_alloc_supplement + old_bytes_loaned_for_young_evac; + assert(allocation_supplement % ShenandoahHeapRegion::region_size_bytes() == 0, + "allocation_supplement must be multiple of region size"); + heap->set_alloc_supplement_reserve(allocation_supplement); // TODO: young_available, which feeds into alloc_budget_evac_and_update is lacking memory available within // existing young-gen regions that were not selected for the collection set. Add this in and adjust the // log message (where it says "empty-region allocation budget"). + log_debug(gc)("Memory reserved for young evacuation: " SIZE_FORMAT "%s for evacuating " SIZE_FORMAT "%s out of young available: " SIZE_FORMAT "%s", byte_size_in_proper_unit(young_evacuated_reserve_used), @@ -979,6 +983,8 @@ size_t ShenandoahGeneration::available() const { } size_t ShenandoahGeneration::adjust_available(intptr_t adjustment) { + assert(adjustment % ShenandoahHeapRegion::region_size_bytes() == 0, + "Adjustment to generation size must be multiple of region size"); _adjusted_capacity = soft_max_capacity() + adjustment; return _adjusted_capacity; } From 0a7d89340e9d543d95805ac402ca0fd9e8a52d0c Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Wed, 1 Feb 2023 18:48:46 +0000 Subject: [PATCH 184/254] Fullgc should honor aging cycle Reviewed-by: wkemper --- src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp | 4 ++++ src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index 21914c94e3f7f..e52f4099f8973 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -289,6 +289,10 @@ void ShenandoahControlThread::run_service() { break; } case stw_full: { + if (age_period-- == 0) { + heap->set_aging_cycle(true); + age_period = ShenandoahAgingCyclePeriod - 1; + } service_stw_full_cycle(cause); break; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index 767a08e7a5a2b..9d110d7fb9f24 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -597,7 +597,11 @@ class ShenandoahPrepareForGenerationalCompactionObjectClosure : public ObjectClo // After full gc compaction, all regions have age 0. Embed the region's age into the object's age in order to preserve // tenuring progress. - _heap->increase_object_age(p, from_region_age + 1); + if (_heap->is_aging_cycle()) { + _heap->increase_object_age(p, from_region_age + 1); + } else { + _heap->increase_object_age(p, from_region_age); + } if (_young_compact_point + obj_size > _young_to_region->end()) { ShenandoahHeapRegion* new_to_region; From 3e8f765726944764a38c489fd8cdc7e72ace290b Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Wed, 1 Feb 2023 18:50:23 +0000 Subject: [PATCH 185/254] Following abbreviated cycle, only increment region age if aging cycle Reviewed-by: wkemper --- src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp index 7d4fd49d0243e..07dab323c1a65 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp @@ -91,7 +91,7 @@ void VM_ShenandoahFinalRoots::doit() { HeapWord* top = r->top(); if (top > tams) { r->reset_age(); - } else { + } else if (heap->is_aging_cycle()){ r->increment_age(); } } From d40ba7dd128a79df0a5d1401327c695ba800fed6 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Wed, 1 Feb 2023 20:07:19 +0000 Subject: [PATCH 186/254] Bootstrap old gc should honor aging cycle Reviewed-by: wkemper --- .../share/gc/shenandoah/shenandoahControlThread.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index e52f4099f8973..80ca08f298931 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -273,7 +273,12 @@ void ShenandoahControlThread::run_service() { { switch (_mode) { case concurrent_normal: { - if ((generation == YOUNG) && (age_period-- == 0)) { + // At this point: + // if (generation == YOUNG), this is a normal YOUNG cycle + // if (generation == OLD), this is a bootstrap OLD cycle + // if (generation == GLOBAL), this is a GLOBAL cycle triggered by System.gc() + // In all three cases, we want to age old objects if this is an aging cycle + if (age_period-- == 0) { heap->set_aging_cycle(true); age_period = ShenandoahAgingCyclePeriod - 1; } From 4ec2cd904476c97d98906ee5b623c57dba261ddb Mon Sep 17 00:00:00 2001 From: William Kemper Date: Wed, 1 Feb 2023 20:10:08 +0000 Subject: [PATCH 187/254] Update vestigial comment Reviewed-by: ysr, kdnilsen --- src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index 7a63cd0d3d060..fc4c6ea905e47 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -115,9 +115,8 @@ class ShenandoahGeneration : public CHeapObj { void reset_bytes_allocated_since_gc_start(); void increase_allocated(size_t bytes); - // Changing the size of the generation will reset the times learned for the heuristic. The heuristic will need to - // relearn collection performance metrics. This also has the effect of preventing further capacity changes from the - // heuristics until at least ShenandoahLearningSteps(5) number of cycles has completed. + // These methods change the capacity of the region by adding or subtracting the given number of bytes from the current + // capacity. void increase_capacity(size_t increment); void decrease_capacity(size_t decrement); From 75811f964674902f8e12ffe255479b75bef6b6e9 Mon Sep 17 00:00:00 2001 From: "Y. Srinivas Ramakrishna" Date: Thu, 2 Feb 2023 18:14:56 +0000 Subject: [PATCH 188/254] 8299703: GenShen: improvements in card scanning Reviewed-by: kdnilsen, wkemper --- .../gc/shenandoah/shenandoahCardStats.cpp | 87 +-- .../gc/shenandoah/shenandoahCardStats.hpp | 142 ++-- .../gc/shenandoah/shenandoahGeneration.cpp | 17 + .../share/gc/shenandoah/shenandoahHeap.cpp | 4 +- .../gc/shenandoah/shenandoahHeapRegion.cpp | 2 +- .../gc/shenandoah/shenandoahHeapRegion.hpp | 8 +- .../gc/shenandoah/shenandoahMarkBitMap.cpp | 3 +- .../shenandoah/shenandoahMarkingContext.hpp | 18 +- .../shenandoahMarkingContext.inline.hpp | 22 +- .../shenandoah/shenandoahScanRemembered.cpp | 15 +- .../shenandoah/shenandoahScanRemembered.hpp | 146 ++-- .../shenandoahScanRemembered.inline.hpp | 628 +++++++++++------- 12 files changed, 580 insertions(+), 512 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCardStats.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCardStats.cpp index e87c1ae75cee1..cb06d99edbacf 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCardStats.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCardStats.cpp @@ -30,96 +30,13 @@ #include "logging/log.hpp" #ifndef PRODUCT -void ShenandoahCardStats::update_run_work(bool record) { - assert(!(_last_dirty || _last_clean) || (_last_dirty && _dirty_run > 0) || (_last_clean && _clean_run > 0), - "dirty/clean run stats inconsistent"); - assert(_dirty_run == 0 || _clean_run == 0, "Both shouldn't be non-zero"); - if (_dirty_run > _max_dirty_run) { - assert(_last_dirty, "Error"); - _max_dirty_run = _dirty_run; - } else if (_clean_run > _max_clean_run) { - assert(_last_clean, "Error"); - _max_clean_run = _clean_run; - } - _dirty_card_cnt += _dirty_run; - _clean_card_cnt += _clean_run; - - // Update local stats - { - assert(_dirty_run <= _cards_in_cluster, "Error"); - assert(_clean_run <= _cards_in_cluster, "Error"); - // Update global stats for distribution of dirty/clean run lengths - _local_card_stats[DIRTY_RUN].add((double)_dirty_run*100/(double)_cards_in_cluster); - _local_card_stats[CLEAN_RUN].add((double)_clean_run*100/(double)_cards_in_cluster); - - if (record) { - // Update global stats for distribution of dirty/clean cards as a percentage of chunk - _local_card_stats[DIRTY_CARDS].add((double)_dirty_card_cnt*100/(double)_cards_in_cluster); - _local_card_stats[CLEAN_CARDS].add((double)_clean_card_cnt*100/(double)_cards_in_cluster); - - // Update global stats for max dirty/clean run distribution as a percentage of chunk - _local_card_stats[MAX_DIRTY_RUN].add((double)_max_dirty_run*100/(double)_cards_in_cluster); - _local_card_stats[MAX_CLEAN_RUN].add((double)_max_clean_run*100/(double)_cards_in_cluster); - - // Update global stats for dirty & clean object counts - _local_card_stats[DIRTY_OBJS].add(_dirty_obj_cnt); - _local_card_stats[CLEAN_OBJS].add(_clean_obj_cnt); - _local_card_stats[DIRTY_SCANS].add(_dirty_scan_cnt); - _local_card_stats[CLEAN_SCANS].add(_clean_scan_cnt); - - _local_card_stats[ALTERNATIONS].add(_alternation_cnt); - } - } - - if (record) { - // reset the stats for the next cluster - _dirty_card_cnt = 0; - _clean_card_cnt = 0; - - _max_dirty_run = 0; - _max_clean_run = 0; - - _dirty_obj_cnt = 0; - _clean_obj_cnt = 0; - - _dirty_scan_cnt = 0; - _clean_scan_cnt = 0; - - _alternation_cnt = 0; - } - _dirty_run = 0; - _clean_run = 0; - _last_dirty = false; - _last_clean = false; - assert(!record || is_clean(), "Error"); -} - -bool ShenandoahCardStats::is_clean() { - return - _dirty_card_cnt == 0 && - _clean_card_cnt == 0 && - _max_dirty_run == 0 && - _max_clean_run == 0 && - _dirty_obj_cnt == 0 && - _clean_obj_cnt == 0 && - _dirty_scan_cnt == 0 && - _clean_scan_cnt == 0 && - _alternation_cnt == 0 && - _dirty_run == 0 && - _clean_run == 0 && - _last_dirty == false && - _last_clean == false; -} - void ShenandoahCardStats::log() const { if (ShenandoahEnableCardStats) { log_info(gc,remset)("Card stats: dirty " SIZE_FORMAT " (max run: " SIZE_FORMAT ")," " clean " SIZE_FORMAT " (max run: " SIZE_FORMAT ")," - " dirty objs " SIZE_FORMAT ", clean objs " SIZE_FORMAT "," - " dirty scans " SIZE_FORMAT ", clean scans " SIZE_FORMAT, + " dirty scans/objs " SIZE_FORMAT, _dirty_card_cnt, _max_dirty_run, _clean_card_cnt, _max_clean_run, - _dirty_obj_cnt, _clean_obj_cnt, - _dirty_scan_cnt, _clean_scan_cnt); + _dirty_scan_obj_cnt); } } #endif // !PRODUCT diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCardStats.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCardStats.hpp index 146348b78348b..abdc6f752049e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCardStats.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCardStats.hpp @@ -29,14 +29,29 @@ #include "gc/shared/gc_globals.hpp" #include "gc/shenandoah/shenandoahNumberSeq.hpp" +enum CardStatType { + DIRTY_RUN = 0, + CLEAN_RUN = 1, + DIRTY_CARDS = 2, + CLEAN_CARDS = 3, + MAX_DIRTY_RUN = 4, + MAX_CLEAN_RUN = 5, + DIRTY_SCAN_OBJS = 6, + ALTERNATIONS = 7, + MAX_CARD_STAT_TYPE = 8 +}; + +enum CardStatLogType { + CARD_STAT_SCAN_RS = 0, + CARD_STAT_UPDATE_REFS = 1, + MAX_CARD_STAT_LOG_TYPE = 2 +}; + class ShenandoahCardStats: public CHeapObj { private: size_t _cards_in_cluster; HdrSeq* _local_card_stats; - bool _last_dirty; - bool _last_clean; - size_t _dirty_card_cnt; size_t _clean_card_cnt; @@ -46,11 +61,7 @@ class ShenandoahCardStats: public CHeapObj { size_t _max_dirty_run; size_t _max_clean_run; - size_t _dirty_obj_cnt; - size_t _clean_obj_cnt; - - size_t _dirty_scan_cnt; - size_t _clean_scan_cnt; + size_t _dirty_scan_obj_cnt; size_t _alternation_cnt; @@ -58,113 +69,68 @@ class ShenandoahCardStats: public CHeapObj { ShenandoahCardStats(size_t cards_in_cluster, HdrSeq* card_stats) : _cards_in_cluster(cards_in_cluster), _local_card_stats(card_stats), - _last_dirty(false), - _last_clean(false), _dirty_card_cnt(0), _clean_card_cnt(0), - _dirty_run(0), - _clean_run(0), _max_dirty_run(0), _max_clean_run(0), - _dirty_obj_cnt(0), - _clean_obj_cnt(0), - _dirty_scan_cnt(0), - _clean_scan_cnt(0), + _dirty_scan_obj_cnt(0), _alternation_cnt(0) { } -private: - void increment_card_cnt_work(bool dirty) { - if (dirty) { // dirty card - if (_last_dirty) { - assert(_dirty_run > 0 && _clean_run == 0 && !_last_clean, "Error"); - _dirty_run++; - } else { - if (_last_clean) { - _alternation_cnt++; - } - update_run(false); - _last_dirty = true; - _dirty_run = 1; - } - } else { // clean card - if (_last_clean) { - assert(_clean_run > 0 && _dirty_run == 0 && !_last_dirty, "Error"); - _clean_run++; - } else { - if (_last_dirty) { - _alternation_cnt++; - } - update_run(false); - _last_clean = true; - _clean_run = 1; - } - } - } + ~ShenandoahCardStats() { + record(); + } - inline void increment_obj_cnt_work(bool dirty) { - assert(!dirty || (_last_dirty && _dirty_run > 0), "Error"); - assert(dirty || (_last_clean && _clean_run > 0), "Error"); - dirty ? _dirty_obj_cnt++ : _clean_obj_cnt++; - } + void record() { + if (ShenandoahEnableCardStats) { + // Update global stats for distribution of dirty/clean cards as a percentage of chunk + _local_card_stats[DIRTY_CARDS].add((double)_dirty_card_cnt*100/(double)_cards_in_cluster); + _local_card_stats[CLEAN_CARDS].add((double)_clean_card_cnt*100/(double)_cards_in_cluster); - inline void increment_scan_cnt_work(bool dirty) { - assert(!dirty || (_last_dirty && _dirty_run > 0), "Error"); - assert(dirty || (_last_clean && _clean_run > 0), "Error"); - dirty ? _dirty_scan_cnt++ : _clean_scan_cnt++; - } + // Update global stats for max dirty/clean run distribution as a percentage of chunk + _local_card_stats[MAX_DIRTY_RUN].add((double)_max_dirty_run*100/(double)_cards_in_cluster); + _local_card_stats[MAX_CLEAN_RUN].add((double)_max_clean_run*100/(double)_cards_in_cluster); - void update_run_work(bool cluster) PRODUCT_RETURN; + // Update global stats for dirty obj scan counts + _local_card_stats[DIRTY_SCAN_OBJS].add(_dirty_scan_obj_cnt); -public: - inline void increment_card_cnt(bool dirty) { - if (ShenandoahEnableCardStats) { - increment_card_cnt_work(dirty); + // Update global stats for alternation counts + _local_card_stats[ALTERNATIONS].add(_alternation_cnt); } } - inline void increment_obj_cnt(bool dirty) { +public: + inline void record_dirty_run(size_t len) { if (ShenandoahEnableCardStats) { - increment_obj_cnt_work(dirty); + _alternation_cnt++; + if (len > _max_dirty_run) { + _max_dirty_run = len; + } + _dirty_card_cnt += len; + assert(len <= _cards_in_cluster, "Error"); + _local_card_stats[DIRTY_RUN].add((double)len*100.0/(double)_cards_in_cluster); } } - inline void increment_scan_cnt(bool dirty) { + inline void record_clean_run(size_t len) { if (ShenandoahEnableCardStats) { - increment_scan_cnt_work(dirty); + _alternation_cnt++; + if (len > _max_clean_run) { + _max_clean_run = len; + } + _clean_card_cnt += len; + assert(len <= _cards_in_cluster, "Error"); + _local_card_stats[CLEAN_RUN].add((double)len*100.0/(double)_cards_in_cluster); } } - inline void update_run(bool record) { + inline void record_scan_obj_cnt(size_t i) { if (ShenandoahEnableCardStats) { - update_run_work(record); + _dirty_scan_obj_cnt += i; } } - bool is_clean() PRODUCT_RETURN0; - void log() const PRODUCT_RETURN; }; -enum CardStatType { - DIRTY_RUN = 0, - CLEAN_RUN = 1, - DIRTY_CARDS = 2, - CLEAN_CARDS = 3, - MAX_DIRTY_RUN = 4, - MAX_CLEAN_RUN = 5, - DIRTY_OBJS = 6, - CLEAN_OBJS = 7, - DIRTY_SCANS = 8, - CLEAN_SCANS= 9, - ALTERNATIONS = 10, - MAX_CARD_STAT_TYPE = 11 -}; - -enum CardStatLogType { - CARD_STAT_SCAN_RS = 0, - CARD_STAT_UPDATE_REFS = 1, - MAX_CARD_STAT_LOG_TYPE = 2 -}; - #endif // SHARE_GC_SHENANDOAH_SHENANDOAHCARDSTATS_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index ddfa10f6d4d4f..dcf5625ca5ffa 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -983,6 +983,11 @@ size_t ShenandoahGeneration::available() const { } size_t ShenandoahGeneration::adjust_available(intptr_t adjustment) { + // TODO: ysr: remove this check & warning + if (adjustment % ShenandoahHeapRegion::region_size_bytes() != 0) { + log_warning(gc)("Adjustment (" INTPTR_FORMAT ") should be a multiple of region size (" SIZE_FORMAT ")", + adjustment, ShenandoahHeapRegion::region_size_bytes()); + } assert(adjustment % ShenandoahHeapRegion::region_size_bytes() == 0, "Adjustment to generation size must be multiple of region size"); _adjusted_capacity = soft_max_capacity() + adjustment; @@ -1015,6 +1020,12 @@ size_t ShenandoahGeneration::adjusted_unaffiliated_regions() const { void ShenandoahGeneration::increase_capacity(size_t increment) { shenandoah_assert_heaplocked_or_safepoint(); assert(_max_capacity + increment <= ShenandoahHeap::heap()->max_size_for(this), "Cannot increase generation capacity beyond maximum."); + assert(increment % ShenandoahHeapRegion::region_size_bytes() == 0, "Region-sized changes only"); + // TODO: ysr: remove this check and warning + if (increment % ShenandoahHeapRegion::region_size_bytes() != 0) { + log_warning(gc)("Increment (" INTPTR_FORMAT ") should be a multiple of region size (" SIZE_FORMAT ")", + increment, ShenandoahHeapRegion::region_size_bytes()); + } _max_capacity += increment; _soft_max_capacity += increment; _adjusted_capacity += increment; @@ -1023,6 +1034,12 @@ void ShenandoahGeneration::increase_capacity(size_t increment) { void ShenandoahGeneration::decrease_capacity(size_t decrement) { shenandoah_assert_heaplocked_or_safepoint(); assert(_max_capacity - decrement >= ShenandoahHeap::heap()->min_size_for(this), "Cannot decrease generation capacity beyond minimum."); + assert(decrement % ShenandoahHeapRegion::region_size_bytes() == 0, "Region-sized changes only"); + // TODO: ysr: remove this check and warning + if (decrement % ShenandoahHeapRegion::region_size_bytes() != 0) { + log_warning(gc)("Decrement (" INTPTR_FORMAT ") should be a multiple of region size (" SIZE_FORMAT ")", + decrement, ShenandoahHeapRegion::region_size_bytes()); + } _max_capacity -= decrement; _soft_max_capacity -= decrement; _adjusted_capacity -= decrement; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 959c01da906b1..7bc1a0af43f31 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -2709,7 +2709,7 @@ class ShenandoahUpdateHeapRefsTask : public WorkerTask { if (r->is_humongous()) { if (start_of_range < end_of_range) { // Need to examine both dirty and clean cards during mixed evac. - r->oop_iterate_humongous_slice(&cl, false, start_of_range, assignment._chunk_size, true, CONCURRENT); + r->oop_iterate_humongous_slice(&cl, false, start_of_range, assignment._chunk_size, true); } } else { // Since this is mixed evacuation, old regions that are candidates for collection have not been coalesced @@ -2779,7 +2779,7 @@ class ShenandoahUpdateHeapRefsTask : public WorkerTask { CardTable::card_size_in_words() * ShenandoahCardCluster::CardsPerCluster; size_t clusters = assignment._chunk_size / cluster_size; assert(clusters * cluster_size == assignment._chunk_size, "Chunk assignment must align on cluster boundaries"); - scanner->process_region_slice(r, assignment._chunk_offset, clusters, end_of_range, &cl, true, CONCURRENT, worker_id); + scanner->process_region_slice(r, assignment._chunk_offset, clusters, end_of_range, &cl, true, worker_id); } } if (ShenandoahPacing && (start_of_range < end_of_range)) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index d5c7b3cb1a1e0..effcede6ac67e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -590,7 +590,7 @@ void ShenandoahHeapRegion::global_oop_iterate_objects_and_fill_dead(OopIterateCl // DO NOT CANCEL. If this worker thread has accepted responsibility for scanning a particular range of addresses, it // must finish the work before it can be cancelled. void ShenandoahHeapRegion::oop_iterate_humongous_slice(OopIterateClosure* blk, bool dirty_only, - HeapWord* start, size_t words, bool write_table, bool is_concurrent) { + HeapWord* start, size_t words, bool write_table) { assert(words % CardTable::card_size_in_words() == 0, "Humongous iteration must span whole number of cards"); assert(is_humongous(), "only humongous region here"); ShenandoahHeap* heap = ShenandoahHeap::heap(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp index 824dd22e7082f..b1496ea5b83db 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp @@ -406,8 +406,7 @@ class ShenandoahHeapRegion { // Invoke closure on every reference contained within the humongous object that spans this humongous // region if the reference is contained within a DIRTY card and the reference is no more than words following // start within the humongous object. - void oop_iterate_humongous_slice(OopIterateClosure* cl, bool dirty_only, HeapWord* start, size_t words, - bool write_table, bool is_concurrent); + void oop_iterate_humongous_slice(OopIterateClosure* cl, bool dirty_only, HeapWord* start, size_t words, bool write_table); HeapWord* block_start(const void* p) const; size_t block_size(const HeapWord* p) const; @@ -429,6 +428,11 @@ class ShenandoahHeapRegion { size_t used() const { return byte_size(bottom(), top()); } size_t free() const { return byte_size(top(), end()); } + // Does this region contain this address? + bool contains(HeapWord* p) const { + return (bottom() <= p) && (p < top()); + } + inline void adjust_alloc_metadata(ShenandoahAllocRequest::Type type, size_t); void reset_alloc_metadata(); size_t get_shared_allocs() const; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkBitMap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkBitMap.cpp index c81137298e540..a3846b81c3aab 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkBitMap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkBitMap.cpp @@ -65,7 +65,8 @@ HeapWord* ShenandoahMarkBitMap::get_next_marked_addr(const HeapWord* addr, ShenandoahMarkingContext* ctx = heap->marking_context(); HeapWord* tams = ctx->top_at_mark_start(r); assert(limit != NULL, "limit must not be NULL"); - assert(limit <= tams, "limit must be less than TAMS"); + assert(limit <= r->top(), "limit must be less than top"); + assert(addr <= tams, "addr must be less than TAMS"); #endif // Round addr up to a possible object boundary to be safe. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.hpp index 05948d6aff92b..2c3b51eaae0f4 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.hpp @@ -59,18 +59,18 @@ class ShenandoahMarkingContext : public CHeapObj { inline bool mark_weak(oop obj); // Simple versions of marking accessors, to be used outside of marking (e.g. no possible concurrent updates) - inline bool is_marked(oop) const; - inline bool is_marked_strong(oop obj) const; - inline bool is_marked_weak(oop obj) const; - inline bool is_marked_or_old(oop obj) const; - inline bool is_marked_strong_or_old(oop obj) const; + inline bool is_marked(const oop) const; + inline bool is_marked_strong(const oop obj) const; + inline bool is_marked_weak(const oop obj) const; + inline bool is_marked_or_old(const oop obj) const; + inline bool is_marked_strong_or_old(const oop obj) const; - inline HeapWord* get_next_marked_addr(HeapWord* addr, HeapWord* limit) const; + inline HeapWord* get_next_marked_addr(const HeapWord* addr, const HeapWord* limit) const; - inline bool allocated_after_mark_start(oop obj) const; - inline bool allocated_after_mark_start(HeapWord* addr) const; + inline bool allocated_after_mark_start(const oop obj) const; + inline bool allocated_after_mark_start(const HeapWord* addr) const; - inline HeapWord* top_at_mark_start(ShenandoahHeapRegion* r) const; + inline HeapWord* top_at_mark_start(const ShenandoahHeapRegion* r) const; inline void capture_top_at_mark_start(ShenandoahHeapRegion* r); inline void reset_top_at_mark_start(ShenandoahHeapRegion* r); void initialize_top_at_mark_start(ShenandoahHeapRegion* r); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.inline.hpp index 90a6ca2345bf1..0433b4b1f5360 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.inline.hpp @@ -37,39 +37,39 @@ inline bool ShenandoahMarkingContext::mark_weak(oop obj) { return !allocated_after_mark_start(obj) && _mark_bit_map.mark_weak(cast_from_oop(obj)); } -inline bool ShenandoahMarkingContext::is_marked(oop obj) const { +inline bool ShenandoahMarkingContext::is_marked(const oop obj) const { return allocated_after_mark_start(obj) || _mark_bit_map.is_marked(cast_from_oop(obj)); } -inline bool ShenandoahMarkingContext::is_marked_strong(oop obj) const { +inline bool ShenandoahMarkingContext::is_marked_strong(const oop obj) const { return allocated_after_mark_start(obj) || _mark_bit_map.is_marked_strong(cast_from_oop(obj)); } -inline bool ShenandoahMarkingContext::is_marked_weak(oop obj) const { +inline bool ShenandoahMarkingContext::is_marked_weak(const oop obj) const { return allocated_after_mark_start(obj) || _mark_bit_map.is_marked_weak(cast_from_oop(obj)); } -inline bool ShenandoahMarkingContext::is_marked_or_old(oop obj) const { +inline bool ShenandoahMarkingContext::is_marked_or_old(const oop obj) const { return is_marked(obj) || ShenandoahHeap::heap()->is_old(obj); } -inline bool ShenandoahMarkingContext::is_marked_strong_or_old(oop obj) const { +inline bool ShenandoahMarkingContext::is_marked_strong_or_old(const oop obj) const { return is_marked_strong(obj) || ShenandoahHeap::heap()->is_old(obj); } -inline HeapWord* ShenandoahMarkingContext::get_next_marked_addr(HeapWord* start, HeapWord* limit) const { +inline HeapWord* ShenandoahMarkingContext::get_next_marked_addr(const HeapWord* start, const HeapWord* limit) const { return _mark_bit_map.get_next_marked_addr(start, limit); } -inline bool ShenandoahMarkingContext::allocated_after_mark_start(oop obj) const { - HeapWord* addr = cast_from_oop(obj); +inline bool ShenandoahMarkingContext::allocated_after_mark_start(const oop obj) const { + const HeapWord* addr = cast_from_oop(obj); return allocated_after_mark_start(addr); } -inline bool ShenandoahMarkingContext::allocated_after_mark_start(HeapWord* addr) const { +inline bool ShenandoahMarkingContext::allocated_after_mark_start(const HeapWord* addr) const { uintx index = ((uintx) addr) >> ShenandoahHeapRegion::region_size_bytes_shift(); HeapWord* top_at_mark_start = _top_at_mark_starts[index]; - bool alloc_after_mark_start = addr >= top_at_mark_start; + const bool alloc_after_mark_start = addr >= top_at_mark_start; return alloc_after_mark_start; } @@ -112,7 +112,7 @@ inline void ShenandoahMarkingContext::reset_top_at_mark_start(ShenandoahHeapRegi _top_at_mark_starts_base[r->index()] = r->bottom(); } -inline HeapWord* ShenandoahMarkingContext::top_at_mark_start(ShenandoahHeapRegion* r) const { +inline HeapWord* ShenandoahMarkingContext::top_at_mark_start(const ShenandoahHeapRegion* r) const { return _top_at_mark_starts_base[r->index()]; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp index 6d6441cbaf3cf..bf36ef1f7d7a1 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp @@ -96,7 +96,7 @@ void ShenandoahScanRememberedTask::do_work(uint worker_id) { if (end_of_range > region->top()) { end_of_range = region->top(); } - scanner->process_region_slice(region, assignment._chunk_offset, clusters, end_of_range, &cl, false, _is_concurrent, worker_id); + scanner->process_region_slice(region, assignment._chunk_offset, clusters, end_of_range, &cl, false, worker_id); } #ifdef ENABLE_REMEMBERED_SET_CANCELLATION // This check is currently disabled to avoid crashes that occur @@ -112,9 +112,9 @@ void ShenandoahScanRememberedTask::do_work(uint worker_id) { } size_t ShenandoahRegionChunkIterator::calc_regular_group_size() { - // The group size is calculated from the number of regions. Suppose the entire heap size is N. The first group processes - // N/2 of total heap size. The second group processes N/4 of total heap size. The third group processes N/2 of total heap - // size, and so on. Note that N/2 + N/4 + N/8 + N/16 + ... sums to N if expanded to infinite terms. + // The group size is calculated from the number of regions. Suppose the heap has N regions. The first group processes + // N/2 regions. The second group processes N/4 regions, the third group N/8 regions and so on. + // Note that infinite series N/2 + N/4 + N/8 + N/16 + ... sums to N. // // The normal group size is the number of regions / 2. // @@ -123,7 +123,7 @@ size_t ShenandoahRegionChunkIterator::calc_regular_group_size() { // // The last group also has more than the normal entries because it finishes the total scanning effort. The chunk sizes are // different for each group. The intention is that the first group processes roughly half of the heap, the second processes - // a quarter of the remaining heap, the third processes an eight of what remains and so on. The smallest chunk size + // half of the remaining heap, the third processes half of what remains and so on. The smallest chunk size // is represented by _smallest_chunk_size_words. We do not divide work any smaller than this. // @@ -316,3 +316,8 @@ ShenandoahRegionChunkIterator::ShenandoahRegionChunkIterator(ShenandoahHeap* hea void ShenandoahRegionChunkIterator::reset() { _index = 0; } + +ShenandoahVerifyNoYoungRefsClosure::ShenandoahVerifyNoYoungRefsClosure(): + _heap(ShenandoahHeap::heap()) { + assert(_heap->mode()->is_generational(), "Don't use when non-generational"); +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp index b372d1cc17910..0f77319cee63a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp @@ -192,6 +192,7 @@ class ShenandoahRegionIterator; class ShenandoahMarkingContext; class CardTable; +typedef CardTable::CardValue CardValue; class ShenandoahDirectCardMarkRememberedSet: public CHeapObj { @@ -201,7 +202,6 @@ class ShenandoahDirectCardMarkRememberedSet: public CHeapObj { // CardTable::card_shift = 9; // CardTable::card_size = 512; // CardTable::card_size_in_words = 64; - // CardTable::clean_card_val() // CardTable::dirty_card_val() @@ -211,32 +211,34 @@ class ShenandoahDirectCardMarkRememberedSet: public CHeapObj { size_t _total_card_count; size_t _cluster_count; HeapWord *_whole_heap_base; // Points to first HeapWord of data contained within heap memory - uint8_t *_byte_map; // Points to first entry within the card table - uint8_t *_byte_map_base; // Points to byte_map minus the bias computed from address of heap memory + CardValue* _byte_map; // Points to first entry within the card table + CardValue* _byte_map_base; // Points to byte_map minus the bias computed from address of heap memory public: + // count is the number of cards represented by the card table. ShenandoahDirectCardMarkRememberedSet(ShenandoahCardTable *card_table, size_t total_card_count); ~ShenandoahDirectCardMarkRememberedSet(); // Card index is zero-based relative to _byte_map. - size_t last_valid_index(); - size_t total_cards(); - size_t card_index_for_addr(HeapWord *p); - HeapWord *addr_for_card_index(size_t card_index); - bool is_card_dirty(size_t card_index); - bool is_write_card_dirty(size_t card_index); - void mark_card_as_dirty(size_t card_index); - void mark_range_as_dirty(size_t card_index, size_t num_cards); - void mark_card_as_clean(size_t card_index); - void mark_read_card_as_clean(size_t card_index); - void mark_range_as_clean(size_t card_index, size_t num_cards); - bool is_card_dirty(HeapWord *p); - void mark_card_as_dirty(HeapWord *p); - void mark_range_as_dirty(HeapWord *p, size_t num_heap_words); - void mark_card_as_clean(HeapWord *p); - void mark_range_as_clean(HeapWord *p, size_t num_heap_words); - size_t cluster_count(); + size_t last_valid_index() const; + size_t total_cards() const; + size_t card_index_for_addr(HeapWord *p) const; + HeapWord *addr_for_card_index(size_t card_index) const; + inline const CardValue* get_card_table_byte_map(bool write_table) const; + inline bool is_card_dirty(size_t card_index) const; + inline bool is_write_card_dirty(size_t card_index) const; + inline void mark_card_as_dirty(size_t card_index); + inline void mark_range_as_dirty(size_t card_index, size_t num_cards); + inline void mark_card_as_clean(size_t card_index); + inline void mark_read_card_as_clean(size_t card_index); + inline void mark_range_as_clean(size_t card_index, size_t num_cards); + inline bool is_card_dirty(HeapWord *p) const; + inline void mark_card_as_dirty(HeapWord *p); + inline void mark_range_as_dirty(HeapWord *p, size_t num_heap_words); + inline void mark_card_as_clean(HeapWord *p); + inline void mark_range_as_clean(HeapWord *p, size_t num_heap_words); + inline size_t cluster_count() const; // Called by GC thread at start of concurrent mark to exchange roles of read and write remembered sets. // Not currently used because mutator write barrier does not honor changes to the location of card table. @@ -245,7 +247,7 @@ class ShenandoahDirectCardMarkRememberedSet: public CHeapObj { void merge_write_table(HeapWord* start, size_t word_count) { size_t card_index = card_index_for_addr(start); size_t num_cards = word_count / CardTable::card_size_in_words(); - size_t iterations = num_cards / (sizeof (intptr_t) / sizeof (CardTable::CardValue)); + size_t iterations = num_cards / (sizeof (intptr_t) / sizeof (CardValue)); intptr_t* read_table_ptr = (intptr_t*) &(_card_table->read_byte_map())[card_index]; intptr_t* write_table_ptr = (intptr_t*) &(_card_table->write_byte_map())[card_index]; for (size_t i = 0; i < iterations; i++) { @@ -261,7 +263,7 @@ class ShenandoahDirectCardMarkRememberedSet: public CHeapObj { void reset_remset(HeapWord* start, size_t word_count) { size_t card_index = card_index_for_addr(start); size_t num_cards = word_count / CardTable::card_size_in_words(); - size_t iterations = num_cards / (sizeof (intptr_t) / sizeof (CardTable::CardValue)); + size_t iterations = num_cards / (sizeof (intptr_t) / sizeof (CardValue)); intptr_t* read_table_ptr = (intptr_t*) &(_card_table->read_byte_map())[card_index]; intptr_t* write_table_ptr = (intptr_t*) &(_card_table->write_byte_map())[card_index]; for (size_t i = 0; i < iterations; i++) { @@ -388,7 +390,7 @@ class ShenandoahCardCluster: public CHeapObj { // ObjectStartsInCardRegion bit is set within a crossing_info.offsets.start iff at least one object starts within // a particular card region. We pack this bit into start byte under assumption that start byte is accessed less - // frequently that last byte. This is true when number of clean cards is greater than number of dirty cards. + // frequently than last byte. This is true when number of clean cards is greater than number of dirty cards. static const uint16_t ObjectStartsInCardRegion = 0x80; static const uint16_t FirstStartBits = 0x3f; @@ -404,16 +406,16 @@ class ShenandoahCardCluster: public CHeapObj { object_starts[card_index].offsets.last = value; } - inline void set_has_object_bit(size_t card_index) { + inline void set_starts_object_bit(size_t card_index) { object_starts[card_index].offsets.first |= ObjectStartsInCardRegion; } - inline void clear_has_object_bit(size_t card_index) { + inline void clear_starts_object_bit(size_t card_index) { object_starts[card_index].offsets.first &= ~ObjectStartsInCardRegion; } // Returns true iff an object is known to start within the card memory associated with card card_index. - inline bool has_object(size_t card_index) { + inline bool starts_object(size_t card_index) const { return (object_starts[card_index].offsets.first & ObjectStartsInCardRegion) != 0; } @@ -586,7 +588,7 @@ class ShenandoahCardCluster: public CHeapObj { // // 1. For efficiency, there is no locking in the implementation of register_object() // 2. Thus, it is required that users of this service assure that concurrent/parallel invocations of - // register_object() do pertain to the same card's memory range. See discussion below to undestand + // register_object() do pertain to the same card's memory range. See discussion below to understand // the risks. // 3. When allocating from a TLAB or GCLAB, the mutual exclusion can be guaranteed by assuring that each // LAB's start and end are aligned on card memory boundaries. @@ -611,7 +613,7 @@ class ShenandoahCardCluster: public CHeapObj { // invocations associated with objects that are allocated from "free lists" to provide their own mutual exclusion locking // mechanism. - // Reset the has_object() information to false for all cards in the range between from and to. + // Reset the starts_object() information to false for all cards in the range between from and to. void reset_object_range(HeapWord *from, HeapWord *to); // register_object() requires that the caller hold the heap lock @@ -641,7 +643,7 @@ class ShenandoahCardCluster: public CHeapObj { // start entry either remains the same or it is changed to the start of the coalesced region. // 2. For the card that holds the start of the coalesced object, it will not impact the first start // but it may impact the last start. - // 3. For following cards spanned entirely by the newly coalesced object, it will change has_object + // 3. For following cards spanned entirely by the newly coalesced object, it will change starts_object // to false (and make first-start and last-start "undefined"). // 4. For a following card that is spanned patially by the newly coalesced object, it may change // first-start value, but it will not change the last-start value. @@ -661,18 +663,25 @@ class ShenandoahCardCluster: public CHeapObj { // scan the objects contained therein if the card is dirty // To avoid excessive lookups in a sparse array, the API queries // the card number pertaining to a particular address and then uses the - // card noumber for subsequent information lookups and stores. + // card number for subsequent information lookups and stores. - // If has_object(card_index), this returns the word offset within this card - // memory at which the first object begins. If !has_object(card_index), the - // result is a don't care value. - size_t get_first_start(size_t card_index); + // If starts_object(card_index), this returns the word offset within this card + // memory at which the first object begins. If !starts_object(card_index), the + // result is a don't care value -- asserts in a debug build. + size_t get_first_start(size_t card_index) const; - // If has_object(card_index), this returns the word offset within this card - // memory at which the last object begins. If !has_object(card_index), the + // If starts_object(card_index), this returns the word offset within this card + // memory at which the last object begins. If !starts_object(card_index), the // result is a don't care value. - size_t get_last_start(size_t card_index); + size_t get_last_start(size_t card_index) const; + + // Given a card_index, return the starting address of the first block in the heap + // that straddles into the card. If the card is co-initial with an object, then + // this would return the starting address of the heap that this card covers. + // Expects to be called for a card affiliated with the old generation in + // generational mode. + HeapWord* block_start(size_t card_index) const; }; // ShenandoahScanRemembered is a concrete class representing the @@ -704,15 +713,17 @@ class ShenandoahScanRemembered: public CHeapObj { // Per worker card stats (multiplexed by phase) HdrSeq** _card_stats; + // The types of card metrics that we gather const char* _card_stats_name[MAX_CARD_STAT_TYPE] = { "dirty_run", "clean_run", "dirty_cards", "clean_cards", "max_dirty_run", "max_clean_run", - "dirty_objs", "clean_objs", - "dirty_scans", "clean_scans", + "dirty_scan_objs", "alternations" }; + // The statistics are collected and logged separately for + // card-scans for initial marking, and for updating refs. const char* _card_stat_log_type[MAX_CARD_STAT_LOG_TYPE] = { "Scan Remembered Set", "Update Refs" }; @@ -834,7 +845,7 @@ class ShenandoahScanRemembered: public CHeapObj { void coalesce_objects(HeapWord *addr, size_t length_in_words); HeapWord* first_object_in_card(size_t card_index) { - if (_scc->has_object(card_index)) { + if (_scc->starts_object(card_index)) { return addr_for_card_index(card_index) + _scc->get_first_start(card_index); } else { return nullptr; @@ -848,9 +859,8 @@ class ShenandoahScanRemembered: public CHeapObj { void mark_range_as_empty(HeapWord *addr, size_t length_in_words); // process_clusters() scans a portion of the remembered set - // to scan roots from old gen into young to identify live objects - // in the young generation. Several worker threads scan different - // portions of the remembered set by making parallel invocations + // for references from old gen into young. Several worker threads + // scan different portions of the remembered set by making parallel invocations // of process_clusters() with each invocation scanning different // "clusters" of the remembered set. // @@ -880,20 +890,17 @@ class ShenandoahScanRemembered: public CHeapObj { // All template expansions require methods to be defined in the inline.hpp file, but larger // such methods need not be declared as inline. - template - inline void process_clusters(size_t first_cluster, size_t count, HeapWord *end_of_range, ClosureType *oops, bool is_concurrent, uint worker_id); - template void process_clusters(size_t first_cluster, size_t count, HeapWord *end_of_range, ClosureType *oops, - bool use_write_table, bool is_concurrent, uint worker_id); + bool use_write_table, uint worker_id); template inline void process_humongous_clusters(ShenandoahHeapRegion* r, size_t first_cluster, size_t count, - HeapWord *end_of_range, ClosureType *oops, bool use_write_table, bool is_concurrent); + HeapWord *end_of_range, ClosureType *oops, bool use_write_table); template inline void process_region_slice(ShenandoahHeapRegion* region, size_t offset, size_t clusters, HeapWord* end_of_range, - ClosureType *cl, bool use_write_table, bool is_concurrent, uint worker_id); + ClosureType *cl, bool use_write_table, uint worker_id); // To Do: // Create subclasses of ShenandoahInitMarkRootsClosure and @@ -931,22 +938,26 @@ class ShenandoahScanRemembered: public CHeapObj { void merge_worker_card_stats_cumulative(HdrSeq* worker_stats, HdrSeq* cum_stats) PRODUCT_RETURN; }; + +// A ShenandoahRegionChunk represents a contiguous interval of a ShenandoahHeapRegion, typically representing +// work to be done by a worker thread. struct ShenandoahRegionChunk { - ShenandoahHeapRegion *_r; + ShenandoahHeapRegion *_r; // The region of which this represents a chunk size_t _chunk_offset; // HeapWordSize offset size_t _chunk_size; // HeapWordSize qty }; -// ShenandoahRegionChunkIterator divides the total remembered set scanning effort into assignments (ShenandoahRegionChunks) -// that are assigned one at a time to worker threads. Note that the effort required to scan a range of memory is not -// necessarily a linear function of the size of the range. Some memory ranges hold only a small number of live objects. -// Some ranges hold primarily primitive (non-pointer data). We start with larger assignment sizes because larger assignments -// can be processed with less coordination effort. We expect that the GC worker threads that receive more difficult assignments -// will work longer on those assignments. Meanwhile, other worker will threads repeatedly accept and complete multiple -// easier assignments. As the total amount of work remaining to be completed decreases, we decrease the size of assignments -// given to individual threads. This reduces the likelihood that significant imbalance between worker thread assignments -// will be introduced when there is less meaningful work to be performed by the remaining worker threads while they wait for -// worker threads with difficult assignments to finish. +// ShenandoahRegionChunkIterator divides the total remembered set scanning effort into ShenandoahRegionChunks +// that are assigned one at a time to worker threads. (Here, we use the terms `assignments` and `chunks` +// interchangeably.) Note that the effort required to scan a range of memory is not necessarily a linear +// function of the size of the range. Some memory ranges hold only a small number of live objects. +// Some ranges hold primarily primitive (non-pointer) data. We start with larger chunk sizes because larger chunks +// reduce coordination overhead. We expect that the GC worker threads that receive more difficult assignments +// will work longer on those chunks. Meanwhile, other worker will threads repeatedly accept and complete multiple +// easier chunks. As the total amount of work remaining to be completed decreases, we decrease the size of chunks +// given to individual threads. This reduces the likelihood of significant imbalance between worker thread assignments +// when there is less meaningful work to be performed by the remaining worker threads while they wait for +// worker threads with difficult assignments to finish, reducing the overall duration of the phase. class ShenandoahRegionChunkIterator : public StackObj { private: @@ -959,7 +970,7 @@ class ShenandoahRegionChunkIterator : public StackObj { // smallest_chunk_size is 4 clusters. Each cluster spans 128 KiB. // This is computed from CardTable::card_size_in_words() * // ShenandoahCardCluster::CardsPerCluster; - static size_t smallest_chunk_size_words() { + static const size_t smallest_chunk_size_words() { return _clusters_in_smallest_chunk * CardTable::card_size_in_words() * ShenandoahCardCluster::CardsPerCluster; } @@ -1051,4 +1062,17 @@ class ShenandoahScanRememberedTask : public WorkerTask { void work(uint worker_id); void do_work(uint worker_id); }; + +// Verify that the oop doesn't point into the young generation +class ShenandoahVerifyNoYoungRefsClosure: public BasicOopIterateClosure { + ShenandoahHeap* _heap; + template void work(T* p); + + public: + ShenandoahVerifyNoYoungRefsClosure(); + + virtual void do_oop(narrowOop* p) { work(p); } + virtual void do_oop(oop* p) { work(p); } +}; + #endif // SHARE_GC_SHENANDOAH_SHENANDOAHSCANREMEMBERED_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp index 214f29e214abb..68e2105d72162 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp @@ -37,46 +37,53 @@ #include "gc/shenandoah/shenandoahScanRemembered.hpp" inline size_t -ShenandoahDirectCardMarkRememberedSet::last_valid_index() { +ShenandoahDirectCardMarkRememberedSet::last_valid_index() const { return _card_table->last_valid_index(); } inline size_t -ShenandoahDirectCardMarkRememberedSet::total_cards() { +ShenandoahDirectCardMarkRememberedSet::total_cards() const { return _total_card_count; } inline size_t -ShenandoahDirectCardMarkRememberedSet::card_index_for_addr(HeapWord *p) { +ShenandoahDirectCardMarkRememberedSet::card_index_for_addr(HeapWord *p) const { return _card_table->index_for(p); } -inline HeapWord * -ShenandoahDirectCardMarkRememberedSet::addr_for_card_index(size_t card_index) { +inline HeapWord* +ShenandoahDirectCardMarkRememberedSet::addr_for_card_index(size_t card_index) const { return _whole_heap_base + CardTable::card_size_in_words() * card_index; } +inline const CardValue* +ShenandoahDirectCardMarkRememberedSet::get_card_table_byte_map(bool use_write_table) const { + return use_write_table ? + _card_table->write_byte_map() + : _card_table->read_byte_map(); +} + inline bool -ShenandoahDirectCardMarkRememberedSet::is_write_card_dirty(size_t card_index) { - uint8_t *bp = &(_card_table->write_byte_map())[card_index]; +ShenandoahDirectCardMarkRememberedSet::is_write_card_dirty(size_t card_index) const { + CardValue* bp = &(_card_table->write_byte_map())[card_index]; return (bp[0] == CardTable::dirty_card_val()); } inline bool -ShenandoahDirectCardMarkRememberedSet::is_card_dirty(size_t card_index) { - uint8_t *bp = &(_card_table->read_byte_map())[card_index]; +ShenandoahDirectCardMarkRememberedSet::is_card_dirty(size_t card_index) const { + CardValue* bp = &(_card_table->read_byte_map())[card_index]; return (bp[0] == CardTable::dirty_card_val()); } inline void ShenandoahDirectCardMarkRememberedSet::mark_card_as_dirty(size_t card_index) { - uint8_t *bp = &(_card_table->write_byte_map())[card_index]; + CardValue* bp = &(_card_table->write_byte_map())[card_index]; bp[0] = CardTable::dirty_card_val(); } inline void ShenandoahDirectCardMarkRememberedSet::mark_range_as_dirty(size_t card_index, size_t num_cards) { - uint8_t *bp = &(_card_table->write_byte_map())[card_index]; + CardValue* bp = &(_card_table->write_byte_map())[card_index]; while (num_cards-- > 0) { *bp++ = CardTable::dirty_card_val(); } @@ -84,36 +91,36 @@ ShenandoahDirectCardMarkRememberedSet::mark_range_as_dirty(size_t card_index, si inline void ShenandoahDirectCardMarkRememberedSet::mark_card_as_clean(size_t card_index) { - uint8_t *bp = &(_card_table->write_byte_map())[card_index]; + CardValue* bp = &(_card_table->write_byte_map())[card_index]; bp[0] = CardTable::clean_card_val(); } inline void ShenandoahDirectCardMarkRememberedSet::mark_range_as_clean(size_t card_index, size_t num_cards) { - uint8_t *bp = &(_card_table->write_byte_map())[card_index]; + CardValue* bp = &(_card_table->write_byte_map())[card_index]; while (num_cards-- > 0) { *bp++ = CardTable::clean_card_val(); } } inline bool -ShenandoahDirectCardMarkRememberedSet::is_card_dirty(HeapWord *p) { +ShenandoahDirectCardMarkRememberedSet::is_card_dirty(HeapWord *p) const { size_t index = card_index_for_addr(p); - uint8_t *bp = &(_card_table->read_byte_map())[index]; + CardValue* bp = &(_card_table->read_byte_map())[index]; return (bp[0] == CardTable::dirty_card_val()); } inline void ShenandoahDirectCardMarkRememberedSet::mark_card_as_dirty(HeapWord *p) { size_t index = card_index_for_addr(p); - uint8_t *bp = &(_card_table->write_byte_map())[index]; + CardValue* bp = &(_card_table->write_byte_map())[index]; bp[0] = CardTable::dirty_card_val(); } inline void ShenandoahDirectCardMarkRememberedSet::mark_range_as_dirty(HeapWord *p, size_t num_heap_words) { - uint8_t *bp = &(_card_table->write_byte_map_base())[uintptr_t(p) >> _card_shift]; - uint8_t *end_bp = &(_card_table->write_byte_map_base())[uintptr_t(p + num_heap_words) >> _card_shift]; + CardValue* bp = &(_card_table->write_byte_map_base())[uintptr_t(p) >> _card_shift]; + CardValue* end_bp = &(_card_table->write_byte_map_base())[uintptr_t(p + num_heap_words) >> _card_shift]; // If (p + num_heap_words) is not aligned on card boundary, we also need to dirty last card. if (((unsigned long long) (p + num_heap_words)) & (CardTable::card_size() - 1)) { end_bp++; @@ -126,20 +133,20 @@ ShenandoahDirectCardMarkRememberedSet::mark_range_as_dirty(HeapWord *p, size_t n inline void ShenandoahDirectCardMarkRememberedSet::mark_card_as_clean(HeapWord *p) { size_t index = card_index_for_addr(p); - uint8_t *bp = &(_card_table->write_byte_map())[index]; + CardValue* bp = &(_card_table->write_byte_map())[index]; bp[0] = CardTable::clean_card_val(); } inline void ShenandoahDirectCardMarkRememberedSet::mark_read_card_as_clean(size_t index) { - uint8_t *bp = &(_card_table->read_byte_map())[index]; + CardValue* bp = &(_card_table->read_byte_map())[index]; bp[0] = CardTable::clean_card_val(); } inline void ShenandoahDirectCardMarkRememberedSet::mark_range_as_clean(HeapWord *p, size_t num_heap_words) { - uint8_t *bp = &(_card_table->write_byte_map_base())[uintptr_t(p) >> _card_shift]; - uint8_t *end_bp = &(_card_table->write_byte_map_base())[uintptr_t(p + num_heap_words) >> _card_shift]; + CardValue* bp = &(_card_table->write_byte_map_base())[uintptr_t(p) >> _card_shift]; + CardValue* end_bp = &(_card_table->write_byte_map_base())[uintptr_t(p + num_heap_words) >> _card_shift]; // If (p + num_heap_words) is not aligned on card boundary, we also need to clean last card. if (((unsigned long long) (p + num_heap_words)) & (CardTable::card_size() - 1)) { end_bp++; @@ -150,7 +157,7 @@ ShenandoahDirectCardMarkRememberedSet::mark_range_as_clean(HeapWord *p, size_t n } inline size_t -ShenandoahDirectCardMarkRememberedSet::cluster_count() { +ShenandoahDirectCardMarkRememberedSet::cluster_count() const { return _cluster_count; } @@ -186,8 +193,8 @@ ShenandoahCardCluster::register_object_wo_lock(HeapWord* address) HeapWord *card_start_address = _rs->addr_for_card_index(card_at_start); uint8_t offset_in_card = address - card_start_address; - if (!has_object(card_at_start)) { - set_has_object_bit(card_at_start); + if (!starts_object(card_at_start)) { + set_starts_object_bit(card_at_start); set_first_start(card_at_start, offset_in_card); set_last_start(card_at_start, offset_in_card); } else { @@ -229,18 +236,18 @@ ShenandoahCardCluster::coalesce_objects(HeapWord* address, size_t // All the cards between first and last get cleared. for (size_t i = card_at_start + 1; i < card_at_end; i++) { - clear_has_object_bit(i); + clear_starts_object_bit(i); } uint8_t follow_offset = static_cast((address + length_in_words) - _rs->addr_for_card_index(card_at_end)); - if (has_object(card_at_end) && (get_first_start(card_at_end) < follow_offset)) { + if (starts_object(card_at_end) && (get_first_start(card_at_end) < follow_offset)) { // It may be that after coalescing within this last card's memory range, the last card // no longer holds an object. if (get_last_start(card_at_end) >= follow_offset) { set_first_start(card_at_end, follow_offset); } else { // last_start is being coalesced so this card no longer has any objects. - clear_has_object_bit(card_at_end); + clear_starts_object_bit(card_at_end); } } // else @@ -253,18 +260,91 @@ ShenandoahCardCluster::coalesce_objects(HeapWord* address, size_t template inline size_t -ShenandoahCardCluster::get_first_start(size_t card_index) { - assert(has_object(card_index), "Can't get first start because no object starts here"); +ShenandoahCardCluster::get_first_start(size_t card_index) const { + assert(starts_object(card_index), "Can't get first start because no object starts here"); return object_starts[card_index].offsets.first & FirstStartBits; } template inline size_t -ShenandoahCardCluster::get_last_start(size_t card_index) { - assert(has_object(card_index), "Can't get last start because no object starts here"); +ShenandoahCardCluster::get_last_start(size_t card_index) const { + assert(starts_object(card_index), "Can't get last start because no object starts here"); return object_starts[card_index].offsets.last; } +// Given a card_index, return the starting address of the first block in the heap +// that straddles into this card. If this card is co-initial with an object, then +// this would return the first address of the range that this card covers, which is +// where the card's first object also begins. +// TODO: collect some stats for the size of walks backward over cards. +// For larger objects, a logarithmic BOT such as used by G1 might make the +// backwards walk potentially faster. +template +HeapWord* +ShenandoahCardCluster::block_start(const size_t card_index) const { + + HeapWord* left = _rs->addr_for_card_index(card_index); + +#ifdef ASSERT + assert(ShenandoahHeap::heap()->mode()->is_generational(), "Do not use in non-generational mode"); + ShenandoahHeapRegion* region = ShenandoahHeap::heap()->heap_region_containing(left); + assert(region->is_old(), "Do not use for young regions"); + // For HumongousRegion:s it's more efficient to jump directly to the + // start region. + assert(!region->is_humongous(), "Use region->humongous_start_region() instead"); +#endif + if (starts_object(card_index) && get_first_start(card_index) == 0) { + // This card contains a co-initial object; a fortiori, it covers + // also the case of a card being the first in a region. + assert(oopDesc::is_oop(cast_to_oop(left)), "Should be an object"); + return left; + } + + HeapWord* p = nullptr; + oop obj = cast_to_oop(p); + ssize_t cur_index = (ssize_t)card_index; + assert(cur_index >= 0, "Overflow"); + assert(cur_index > 0, "Should have returned above"); + // Walk backwards over the cards... + while (--cur_index > 0 && !starts_object(cur_index)) { + // ... to the one that starts the object + } + // cur_index should start an object: we should not have walked + // past the left end of the region. + assert(cur_index >= 0 && (cur_index <= (ssize_t)card_index), "Error"); + assert(region->bottom() <= _rs->addr_for_card_index(cur_index), + "Fell off the bottom of containing region"); + assert(starts_object(cur_index), "Error"); + size_t offset = get_last_start(cur_index); + // can avoid call via card size arithmetic below instead + p = _rs->addr_for_card_index(cur_index) + offset; + // Recall that we already dealt with the co-initial object case above + assert(p < left, "obj should start before left"); + // While it is safe to ask an object its size in the loop that + // follows, the (ifdef'd out) loop should never be needed. + // 1. we ask this question only for regions in the old generation + // 2. there is no direct allocation ever by mutators in old generation + // regions. Only GC will ever allocate in old regions, and then + // too only during promotion/evacuation phases. Thus there is no danger + // of races between reading from and writing to the object start array, + // or of asking partially initialized objects their size (in the loop below). + // 3. only GC asks this question during phases when it is not concurrently + // evacuating/promoting, viz. during concurrent root scanning (before + // the evacuation phase) and during concurrent update refs (after the + // evacuation phase) of young collections. This is never called + // during old or global collections. + // 4. Every allocation under TAMS updates the object start array. + NOT_PRODUCT(obj = cast_to_oop(p);) + assert(oopDesc::is_oop(obj), "Should be an object"); +#define WALK_FORWARD_IN_BLOCK_START false + while (WALK_FORWARD_IN_BLOCK_START && p + obj->size() < left) { + p += obj->size(); + } +#undef WALK_FORWARD_IN_BLOCK_START // false + assert(p + obj->size() > left, "obj should end after left"); + return p; +} + template inline size_t ShenandoahScanRemembered::last_valid_index() { return _rs->last_valid_index(); } @@ -348,7 +428,7 @@ inline bool ShenandoahScanRemembered::verify_registration(HeapWord* address, ShenandoahMarkingContext* ctx) { size_t index = card_index_for_addr(address); - if (!_scc->has_object(index)) { + if (!_scc->starts_object(index)) { return false; } HeapWord* base_addr = addr_for_card_index(index); @@ -403,7 +483,7 @@ ShenandoahScanRemembered::verify_registration(HeapWord* address, if (end_card_index > index && end_card_index <= _rs->last_valid_index()) { // If there is a following object registered on the next card, it should begin where this object ends. - if (_scc->has_object(end_card_index) && + if (_scc->starts_object(end_card_index) && ((addr_for_card_index(end_card_index) + _scc->get_first_start(end_card_index)) != (base_addr + offset))) { return false; } @@ -411,7 +491,7 @@ ShenandoahScanRemembered::verify_registration(HeapWord* address, // Assure that no other objects are registered "inside" of this one. for (index++; index < end_card_index; index++) { - if (_scc->has_object(index)) { + if (_scc->starts_object(index)) { return false; } } @@ -462,29 +542,18 @@ ShenandoahScanRemembered::mark_range_as_empty(HeapWord *addr, siz _scc->clear_objects_in_range(addr, length_in_words); } -template -template -inline void ShenandoahScanRemembered::process_clusters(size_t first_cluster, size_t count, HeapWord *end_of_range, - ClosureType *cl, bool is_concurrent, uint worker_id) { - process_clusters(first_cluster, count, end_of_range, cl, false, is_concurrent, worker_id); -} - -// Process all objects starting within count clusters beginning with first_cluster for which the start address is +// Process all objects starting within count clusters beginning with first_cluster and for which the start address is // less than end_of_range. For any non-array object whose header lies on a dirty card, scan the entire object, -// even if its end reaches beyond end_of_range. Object arrays, on the other hand,s are precisely dirtied and only -// the portions of the array on dirty cards need to be scanned. +// even if its end reaches beyond end_of_range. Object arrays, on the other hand, are precisely dirtied and +// only the portions of the array on dirty cards need to be scanned. // -// Do not CANCEL within process_clusters. It is assumed that if a worker thread accepts responsbility for processing +// Do not CANCEL within process_clusters. It is assumed that if a worker thread accepts responsibility for processing // a chunk of work, it will finish the work it starts. Otherwise, the chunk of work will be lost in the transition to -// degenerated execution. +// degenerated execution, leading to dangling references. template template -void ShenandoahScanRemembered::process_clusters(size_t first_cluster, size_t count, HeapWord *end_of_range, - ClosureType *cl, bool write_table, bool is_concurrent, uint worker_id) { - - // Unlike traditional Shenandoah marking, the old-gen resident objects that are examined as part of the remembered set are not - // always themselves marked. Each such object will be scanned exactly once. Any young-gen objects referenced from the remembered - // set will be marked and then subsequently scanned. +void ShenandoahScanRemembered::process_clusters(size_t first_cluster, size_t count, HeapWord* end_of_range, + ClosureType* cl, bool use_write_table, uint worker_id) { // If old-gen evacuation is active, then MarkingContext for old-gen heap regions is valid. We use the MarkingContext // bits to determine which objects within a DIRTY card need to be scanned. This is necessary because old-gen heap @@ -492,189 +561,235 @@ void ShenandoahScanRemembered::process_clusters(size_t first_clus // may contain zombie objects. Zombie objects are known to be dead, but have not yet been "collected". Scanning // zombie objects is unsafe because the Klass pointer is not reliable, objects referenced from a zombie may have been // collected (if dead), or relocated (if live), or if dead but not yet collected, we don't want to "revive" them - // by marking them (when marking) or evacuating them (when updating refereces). - - ShenandoahHeap* heap = ShenandoahHeap::heap(); - ShenandoahMarkingContext* ctx; + // by marking them (when marking) or evacuating them (when updating references). + + // start and end addresses of range of objects to be scanned, clipped to end_of_range + const size_t start_card_index = first_cluster * ShenandoahCardCluster::CardsPerCluster; + const HeapWord* start_addr = _rs->addr_for_card_index(start_card_index); + // clip at end_of_range (exclusive) + HeapWord* end_addr = MIN2(end_of_range, (HeapWord*)start_addr + (count * ShenandoahCardCluster::CardsPerCluster + * CardTable::card_size_in_words())); + assert(start_addr < end_addr, "Empty region?"); + + const size_t whole_cards = (end_addr - start_addr + CardTable::card_size_in_words() - 1)/CardTable::card_size_in_words(); + const size_t end_card_index = start_card_index + whole_cards - 1; + log_debug(gc, remset)("Worker %u: cluster = " SIZE_FORMAT " count = " SIZE_FORMAT " eor = " INTPTR_FORMAT + " start_addr = " INTPTR_FORMAT " end_addr = " INTPTR_FORMAT " cards = " SIZE_FORMAT, + worker_id, first_cluster, count, p2i(end_of_range), p2i(start_addr), p2i(end_addr), whole_cards); + + // use_write_table states whether we are using the card table that is being + // marked by the mutators. If false, we are using a snapshot of the card table + // that is not subject to modifications. Even when this arg is true, and + // the card table is being actively marked, SATB marking ensures that we need not + // worry about cards marked after the processing here has passed them. + const CardValue* const ctbm = _rs->get_card_table_byte_map(use_write_table); + + // If old gen evacuation is active, ctx will hold the completed marking of + // old generation objects. We'll only scan objects that are marked live by + // the old generation marking. These include objects allocated since the + // start of old generation marking (being those above TAMS). + const ShenandoahHeap* heap = ShenandoahHeap::heap(); + const ShenandoahMarkingContext* ctx = heap->is_old_bitmap_stable() ? + heap->marking_context() : nullptr; + + // The region we will scan is the half-open interval [start_addr, end_addr), + // and lies entirely within a single region. + const ShenandoahHeapRegion* region = ShenandoahHeap::heap()->heap_region_containing(start_addr); + assert(region->contains(end_addr - 1), "Slice shouldn't cross regions"); + + // This code may have implicit assumptions of examining only old gen regions. + assert(region->is_old(), "We only expect to be processing old regions"); + assert(!region->is_humongous(), "Humongous regions can be processed more efficiently;" + "see process_humongous_clusters()"); + // tams and ctx below are for old generation marking. As such, young gen roots must + // consider everything above tams, since it doesn't represent a TAMS for young gen's + // SATB marking. + const HeapWord* tams = (ctx == nullptr ? region->bottom() : ctx->top_at_mark_start(region)); + + NOT_PRODUCT(ShenandoahCardStats stats(whole_cards, card_stats(worker_id));) + + // In the case of imprecise marking, we remember the lowest address + // scanned in a range of dirty cards, as we work our way left from the + // highest end_addr. This serves as another upper bound on the address we will + // scan as we move left over each contiguous range of dirty cards. + HeapWord* upper_bound = nullptr; + + // Starting at the right end of the address range, walk backwards accumulating + // a maximal dirty range of cards, then process those cards. + ssize_t cur_index = (ssize_t) end_card_index; + assert(cur_index >= 0, "Overflow"); + assert(((ssize_t)start_card_index) >= 0, "Overflow"); + while (cur_index >= (ssize_t)start_card_index) { + + // We'll continue the search starting with the card for the upper bound + // address identified by the last dirty range that we processed, if any, + // skipping any cards at higher addresses. + if (upper_bound != nullptr) { + ssize_t right_index = _rs->card_index_for_addr(upper_bound); + assert(right_index >= 0, "Overflow"); + cur_index = MIN2(cur_index, right_index); + assert(upper_bound < end_addr, "Program logic"); + end_addr = upper_bound; // lower end_addr + upper_bound = nullptr; // and clear upper_bound + if (end_addr <= start_addr) { + assert(right_index <= (ssize_t)start_card_index, "Program logic"); + // We are done with our cluster + return; + } + } - if (heap->is_old_bitmap_stable()) { - ctx = heap->marking_context(); - } else { - ctx = nullptr; - } + if (ctbm[cur_index] == CardTable::dirty_card_val()) { + // ==== BEGIN DIRTY card range processing ==== - size_t cur_cluster = first_cluster; - size_t cur_count = count; - size_t card_index = cur_cluster * ShenandoahCardCluster::CardsPerCluster; - HeapWord* start_of_range = _rs->addr_for_card_index(card_index); - ShenandoahHeapRegion* r = heap->heap_region_containing(start_of_range); - assert(end_of_range <= r->top(), "process_clusters() examines one region at a time"); - - NOT_PRODUCT(ShenandoahCardStats stats(ShenandoahCardCluster::CardsPerCluster, card_stats(worker_id));) - - while (cur_count-- > 0) { - // TODO: do we want to check cancellation in inner loop, on every card processed? That would be more responsive, - // but require more overhead for checking. - card_index = cur_cluster * ShenandoahCardCluster::CardsPerCluster; - size_t end_card_index = card_index + ShenandoahCardCluster::CardsPerCluster; - cur_cluster++; - size_t next_card_index = 0; - - assert(stats.is_clean(), "Error"); - while (card_index < end_card_index) { // TODO: understand why end_of_range is needed. - if (_rs->addr_for_card_index(card_index) > end_of_range) { - cur_count = 0; - card_index = end_card_index; - break; + const size_t dirty_r = cur_index; // record right end of dirty range (inclusive) + while (--cur_index >= (ssize_t)start_card_index && ctbm[cur_index] == CardTable::dirty_card_val()) { + // walk back over contiguous dirty cards to find left end of dirty range (inclusive) } - bool is_dirty = (write_table)? is_write_card_dirty(card_index): is_card_dirty(card_index); - bool has_object = _scc->has_object(card_index); - NOT_PRODUCT(stats.increment_card_cnt(is_dirty);) - if (is_dirty) { - size_t prev_card_index = card_index; - if (has_object) { - // Scan all objects that start within this card region. - size_t start_offset = _scc->get_first_start(card_index); - HeapWord *p = _rs->addr_for_card_index(card_index); - HeapWord *card_start = p; - HeapWord *endp = p + CardTable::card_size_in_words(); - assert(!r->is_humongous(), "Process humongous regions elsewhere"); - - if (endp > end_of_range) { - endp = end_of_range; - next_card_index = end_card_index; + // [dirty_l, dirty_r] is a "maximal" closed interval range of dirty card indices: + // it may not be maximal if we are using the write_table, because of concurrent + // mutations dirtying the card-table. It may also not be maximal if an upper bound + // was established by the scan of the previous chunk. + const size_t dirty_l = cur_index + 1; // record left end of dirty range (inclusive) + // Check that we identified a boundary on our left + assert(ctbm[dirty_l] == CardTable::dirty_card_val(), "First card in range should be dirty"); + assert(dirty_l == start_card_index || use_write_table + || ctbm[dirty_l - 1] == CardTable::clean_card_val(), + "Interval isn't maximal on the left"); + assert(dirty_r >= dirty_l, "Error"); + assert(ctbm[dirty_r] == CardTable::dirty_card_val(), "Last card in range should be dirty"); + // Record alternations, dirty run length, and dirty card count + NOT_PRODUCT(stats.record_dirty_run(dirty_r - dirty_l + 1);) + + // Find first object that starts this range: + // [left, right) is a maximal right-open interval of dirty cards + HeapWord* left = _rs->addr_for_card_index(dirty_l); // inclusive + HeapWord* right = _rs->addr_for_card_index(dirty_r + 1); // exclusive + // Clip right to end_addr established above (still exclusive) + right = MIN2(right, end_addr); + assert(right <= region->top() && end_addr <= region->top(), "Busted bounds"); + const MemRegion mr(left, right); + + // NOTE: We'll not call block_start() repeatedly + // on a very large object if its head card is dirty. If not, + // (i.e. the head card is clean) we'll call it each time we + // process a new dirty range on the object. This is always + // the case for large object arrays, which are typically more + // common. + // TODO: It is worthwhile to memoize this, so as to avoid that + // overhead, and it is easy to do, but deferred to a follow-up. + HeapWord* p = _scc->block_start(dirty_l); + oop obj = cast_to_oop(p); + + // PREFIX: The object that straddles into this range of dirty cards + // from the left may be subject to special treatment unless + // it is an object array. + if (p < left && !obj->is_objArray()) { + // The mutator (both compiler and interpreter, but not JNI?) + // typically dirty imprecisely (i.e. only the head of an object), + // but GC closures typically dirty the object precisely. (It would + // be nice to have everything be precise for maximum efficiency.) + // + // To handle this, we check the head card of the object here and, + // if dirty, (arrange to) scan the object in its entirety. If we + // find the head card clean, we'll scan only the portion of the + // object lying in the dirty card range below, assuming this was + // the result of precise marking by GC closures. + + // index of the "head card" for p + const size_t hc_index = _rs->card_index_for_addr(p); + if (ctbm[hc_index] == CardTable::dirty_card_val()) { + // Scan or skip the object, depending on location of its + // head card, and remember that we'll have processed all + // the objects back up to p, which is thus an upper bound + // for the next iteration of a dirty card loop. + upper_bound = p; // remember upper bound for next chunk + if (p < start_addr) { + // if object starts in a previous slice, it'll be handled + // in its entirety by the thread processing that slice; we can + // skip over it and avoid an unnecessary extra scan. + assert(obj == cast_to_oop(p), "Inconsistency detected"); + p += obj->size(); } else { - // endp either points to start of next card region, or to the next object that needs to be scanned, which may - // reside in some successor card region. - - // Can't use _scc->card_index_for_addr(endp) here because it crashes with assertion - // failure if endp points to end of heap. - next_card_index = card_index + (endp - card_start) / CardTable::card_size_in_words(); - } - - p += start_offset; - while (p < endp) { - oop obj = cast_to_oop(p); - NOT_PRODUCT(stats.increment_obj_cnt(is_dirty);) - - // ctx->is_marked() returns true if mark bit set or if obj above TAMS. - if (!ctx || ctx->is_marked(obj)) { - // Future TODO: - // For improved efficiency, we might want to give special handling of obj->is_objArray(). In - // particular, in that case, we might want to divide the effort for scanning of a very long object array - // between multiple threads. Also, skip parts of the array that are not marked as dirty. - if (obj->is_objArray()) { - objArrayOop array = objArrayOop(obj); - int len = array->length(); - array->oop_iterate_range(cl, 0, len); - NOT_PRODUCT(stats.increment_scan_cnt(is_dirty);) - } else if (obj->is_instance()) { - obj->oop_iterate(cl); - NOT_PRODUCT(stats.increment_scan_cnt(is_dirty);) - } else { - // Case 3: Primitive array. Do nothing, no oops there. We use the same - // performance tweak TypeArrayKlass::oop_oop_iterate_impl is using: - // We skip iterating over the klass pointer since we know that - // Universe::TypeArrayKlass never moves. - assert (obj->is_typeArray(), "should be type array"); - } - p += obj->size(); + // the object starts in our slice, we scan it in its entirety + assert(obj == cast_to_oop(p), "Inconsistency detected"); + if (ctx == nullptr || ctx->is_marked(obj)) { + // Scan the object in its entirety + p += obj->oop_iterate_size(cl); } else { - // This object is not marked so we don't scan it. Containing region r is initialized above. - HeapWord* tams = ctx->top_at_mark_start(r); - if (p >= tams) { - p += obj->size(); - } else { - p = ctx->get_next_marked_addr(p, tams); - } + assert(p < tams, "Error 1 in ctx/marking/tams logic"); + // Skip over any intermediate dead objects + p = ctx->get_next_marked_addr(p, tams); + assert(p <= tams, "Error 2 in ctx/marking/tams logic"); } } - if (p > endp) { - card_index = card_index + (p - card_start) / CardTable::card_size_in_words(); - } else { // p == endp - card_index = next_card_index; - } + assert(p > left, "Should have processed into interior of dirty range"); + } + } + + size_t i = 0; + HeapWord* last_p = nullptr; + + // BODY: Deal with (other) objects in this dirty card range + while (p < right) { + obj = cast_to_oop(p); + // walk right scanning eligible objects + if (ctx == nullptr || ctx->is_marked(obj)) { + // we need to remember the last object ptr we scanned, in case we need to + // complete a partial suffix scan after mr, see below + last_p = p; + // apply the closure to the oops in the portion of + // the object within mr. + p += obj->oop_iterate_size(cl, mr); + NOT_PRODUCT(i++); } else { - // Card is dirty but has no object. Card will have been scanned during scan of a previous cluster. - card_index++; + // forget the last object pointer we remembered + last_p = nullptr; + assert(p < tams, "Tams and above are implicitly marked in ctx"); + // object under tams isn't marked: skip to next live object + p = ctx->get_next_marked_addr(p, tams); + assert(p <= tams, "Error 3 in ctx/marking/tams logic"); } - } else { - if (has_object) { - // Card is clean but has object. - // Scan the last object that starts within this card memory if it spans at least one dirty card within this cluster - // or if it reaches into the next cluster. - size_t start_offset = _scc->get_last_start(card_index); - HeapWord *card_start = _rs->addr_for_card_index(card_index); - HeapWord *p = card_start + start_offset; - oop obj = cast_to_oop(p); - - size_t last_card; - if (!ctx || ctx->is_marked(obj)) { - HeapWord *nextp = p + obj->size(); - NOT_PRODUCT(stats.increment_obj_cnt(is_dirty);) - - // Can't use _scc->card_index_for_addr(endp) here because it crashes with assertion - // failure if nextp points to end of heap. Must also not attempt to read past last - // valid index for card table. - last_card = card_index + (nextp - card_start) / CardTable::card_size_in_words(); - last_card = MIN2(last_card, last_valid_index()); - - bool reaches_next_cluster = (last_card > end_card_index); - bool spans_dirty_within_this_cluster = false; - - if (!reaches_next_cluster) { - for (size_t span_card = card_index+1; span_card <= last_card; span_card++) { - if ((write_table)? _rs->is_write_card_dirty(span_card): _rs->is_card_dirty(span_card)) { - spans_dirty_within_this_cluster = true; - break; - } - } - } + } - // TODO: only iterate over this object if it spans dirty within this cluster or within following clusters. - // Code as written is known not to examine a zombie object because either the object is marked, or we are - // not using the mark-context to differentiate objects, so the object is known to have been coalesced and - // filled if it is not "live". - - if (reaches_next_cluster || spans_dirty_within_this_cluster) { - if (obj->is_objArray()) { - objArrayOop array = objArrayOop(obj); - int len = array->length(); - array->oop_iterate_range(cl, 0, len); - NOT_PRODUCT(stats.increment_scan_cnt(is_dirty);) - } else if (obj->is_instance()) { - obj->oop_iterate(cl); - NOT_PRODUCT(stats.increment_scan_cnt(is_dirty);) - } else { - // Case 3: Primitive array. Do nothing, no oops there. We use the same - // performance tweak TypeArrayKlass::oop_oop_iterate_impl is using: - // We skip iterating over the klass pointer since we know that - // Universe::TypeArrayKlass never moves. - assert (obj->is_typeArray(), "should be type array"); - } - } - } else { - // The object that spans end of this clean card is not marked, so no need to scan it or its - // unmarked neighbors. Containing region r is initialized above. - HeapWord* tams = ctx->top_at_mark_start(r); - HeapWord* nextp; - if (p >= tams) { - nextp = p + obj->size(); - } else { - nextp = ctx->get_next_marked_addr(p, tams); - } - last_card = card_index + (nextp - card_start) / CardTable::card_size_in_words(); - } - // Increment card_index to account for the spanning object, even if we didn't scan it. - card_index = (last_card > card_index)? last_card: card_index + 1; + // TODO: if an objArray then only use mr, else just iterate over entire object; + // that would avoid the special treatment of suffix below. + + // SUFFIX: Fix up a possible incomplete scan at right end of window + // by scanning the portion of a non-objArray that wasn't done. + if (p > right && last_p != nullptr) { + assert(last_p < right, "Error"); + // check if last_p suffix needs scanning + const oop last_obj = cast_to_oop(last_p); + if (!last_obj->is_objArray()) { + // scan the remaining suffix of the object + const MemRegion last_mr(right, p); + assert(p == last_p + last_obj->size(), "Would miss portion of last_obj"); + last_obj->oop_iterate(cl, last_mr); + log_debug(gc, remset)("Fixed up non-objArray suffix scan in [" INTPTR_FORMAT ", " INTPTR_FORMAT ")", + p2i(last_mr.start()), p2i(last_mr.end())); } else { - // Card is clean and has no object. No need to clean this card. - card_index++; + log_debug(gc, remset)("Skipped suffix scan of objArray in [" INTPTR_FORMAT ", " INTPTR_FORMAT ")", + p2i(right), p2i(p)); } } - } // end of a range of cards in current cluster - NOT_PRODUCT(stats.update_run(true /* record */);) - } // end of all clusters + NOT_PRODUCT(stats.record_scan_obj_cnt(i);) + + // ==== END DIRTY card range processing ==== + } else { + // ==== BEGIN CLEAN card range processing ==== + + assert(ctbm[cur_index] == CardTable::clean_card_val(), "Error"); + // walk back over contiguous clean cards + size_t i = 0; + while (--cur_index >= (ssize_t)start_card_index && ctbm[cur_index] == CardTable::clean_card_val()) { + NOT_PRODUCT(i++); + } + // Record alternations, clean run length, and clean card count + NOT_PRODUCT(stats.record_clean_run(i);) + + // ==== END CLEAN card range processing ==== + } + } } // Given that this range of clusters is known to span a humongous object spanned by region r, scan the @@ -683,8 +798,7 @@ template template inline void ShenandoahScanRemembered::process_humongous_clusters(ShenandoahHeapRegion* r, size_t first_cluster, size_t count, - HeapWord *end_of_range, ClosureType *cl, bool write_table, - bool is_concurrent) { + HeapWord *end_of_range, ClosureType *cl, bool use_write_table) { ShenandoahHeapRegion* start_region = r->humongous_start_region(); HeapWord* p = start_region->bottom(); oop obj = cast_to_oop(p); @@ -695,19 +809,21 @@ ShenandoahScanRemembered::process_humongous_clusters(ShenandoahHe size_t first_card_index = first_cluster * ShenandoahCardCluster::CardsPerCluster; HeapWord* first_cluster_addr = _rs->addr_for_card_index(first_card_index); size_t spanned_words = count * ShenandoahCardCluster::CardsPerCluster * CardTable::card_size_in_words(); - start_region->oop_iterate_humongous_slice(cl, true, first_cluster_addr, spanned_words, write_table, is_concurrent); + start_region->oop_iterate_humongous_slice(cl, true, first_cluster_addr, spanned_words, use_write_table); } + +// This method takes a region & determines the end of the region that the worker can scan. template template inline void ShenandoahScanRemembered::process_region_slice(ShenandoahHeapRegion *region, size_t start_offset, size_t clusters, HeapWord *end_of_range, ClosureType *cl, bool use_write_table, - bool is_concurrent, uint worker_id) { + uint worker_id) { + + // This is called only for young gen collection, when we scan old gen regions + assert(region->is_old(), "Expecting an old region"); HeapWord *start_of_range = region->bottom() + start_offset; - size_t cluster_size = - CardTable::card_size_in_words() * ShenandoahCardCluster::CardsPerCluster; - size_t words = clusters * cluster_size; size_t start_cluster_no = cluster_for_addr(start_of_range); assert(addr_for_cluster(start_cluster_no) == start_of_range, "process_region_slice range must align on cluster boundary"); @@ -736,23 +852,27 @@ ShenandoahScanRemembered::process_region_slice(ShenandoahHeapRegi region->index(), p2i(start_of_range), p2i(end_of_range), use_write_table? "read/write (updating)": "read (marking)"); - // Note that end_of_range may point to the middle of a cluster because region->top() or region->get_update_watermark() may - // be less than start_of_range + words. - - // We want to assure that our process_clusters() request spans all relevant clusters. Note that each cluster - // processed will avoid processing beyond end_of_range. - - // Note that any object that starts between start_of_range and end_of_range, including humongous objects, will - // be fully processed by process_clusters, even though the object may reach beyond end_of_range. - - // If I am assigned to process a range that starts beyond end_of_range (top or update-watermark), we have no work to do. - + // Note that end_of_range may point to the middle of a cluster because we limit scanning to + // region->top() or region->get_update_watermark(). We avoid processing past end_of_range. + // Objects that start between start_of_range and end_of_range, including humongous objects, will + // be fully processed by process_clusters. In no case should we need to scan past end_of_range. if (start_of_range < end_of_range) { if (region->is_humongous()) { ShenandoahHeapRegion* start_region = region->humongous_start_region(); - process_humongous_clusters(start_region, start_cluster_no, clusters, end_of_range, cl, use_write_table, is_concurrent); + // TODO: ysr : This will be called multiple times with same start_region, but different start_cluster_no. + // Check that it does the right thing here, and doesn't do redundant work. Also see if the call API/interface + // can be simplified. + process_humongous_clusters(start_region, start_cluster_no, clusters, end_of_range, cl, use_write_table); } else { - process_clusters(start_cluster_no, clusters, end_of_range, cl, use_write_table, is_concurrent, worker_id); + // TODO: ysr The start_of_range calculated above is discarded and may be calculated again in process_clusters(). + // See if the redundant and wasted calculations can be avoided, and if the call parameters can be cleaned up. + // It almost sounds like this set of methods needs a working class to stash away some useful info that can be + // efficiently passed around amongst these methods, as well as related state. Note that we can't use + // ShenandoahScanRemembered as there seems to be only one instance of that object for the heap which is shared + // by all workers. Note that there are also task methods which call these which may have per worker storage. + // We need to be careful however that if the number of workers changes dynamically that state isn't sequestered + // and become obsolete. + process_clusters(start_cluster_no, clusters, end_of_range, cl, use_write_table, worker_id); } } } @@ -790,9 +910,10 @@ void ShenandoahScanRemembered::roots_do(OopIterateClosure* cl) { // Remembered set scanner if (region->is_humongous()) { process_humongous_clusters(region->humongous_start_region(), start_cluster_no, num_clusters, end_of_range, cl, - false /* is_write_table */, false /* is_concurrent */); + false /* use_write_table */); } else { - process_clusters(start_cluster_no, num_clusters, end_of_range, cl, false /* is_concurrent */, 0); + process_clusters(start_cluster_no, num_clusters, end_of_range, cl, + false /* use_write_table */, 0 /* fake worker id */); } } } @@ -855,20 +976,23 @@ inline bool ShenandoahRegionChunkIterator::has_next() const { } inline bool ShenandoahRegionChunkIterator::next(struct ShenandoahRegionChunk *assignment) { - if (_index > _total_chunks) { + if (_index >= _total_chunks) { return false; } size_t new_index = Atomic::add(&_index, (size_t) 1, memory_order_relaxed); if (new_index > _total_chunks) { + // First worker that hits new_index == _total_chunks continues, other + // contending workers return false. return false; } // convert to zero-based indexing new_index--; + assert(new_index < _total_chunks, "Error"); + // Find the group number for the assigned chunk index size_t group_no; for (group_no = 0; new_index >= _group_entries[group_no]; group_no++) ; - assert(group_no < _num_groups, "Cannot have group no greater or equal to _num_groups"); // All size computations measured in HeapWord @@ -889,4 +1013,14 @@ inline bool ShenandoahRegionChunkIterator::next(struct ShenandoahRegionChunk *as assignment->_chunk_size = group_chunk_size; return true; } + +template +inline void ShenandoahVerifyNoYoungRefsClosure::work(T* p) { + T o = RawAccess<>::oop_load(p); + if (!CompressedOops::is_null(o)) { + oop obj = CompressedOops::decode_not_null(o); + assert(!_heap->is_in_young(obj), "Found a young ref"); + } +} + #endif // SHARE_GC_SHENANDOAH_SHENANDOAHSCANREMEMBEREDINLINE_HPP From 0dbacdfeb7229279a7b5f2ec8075c8bbd71a3a36 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Thu, 16 Feb 2023 17:51:59 +0000 Subject: [PATCH 189/254] Remove unused visualizer option Reviewed-by: kdnilsen --- src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp index d02c949f5d6e6..a664353e97921 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp @@ -241,12 +241,6 @@ "the samples. Higher values provide more fidelity, at expense " \ "of more sampling overhead.") \ \ - product(ccstr, ShenandoahRegionSamplingFile, \ - "./shenandoahSnapshots_pid%p.log", \ - "If ShenandoahLogRegionSampling is on, save sampling data stream "\ - "to this file [default: ./shenandoahSnapshots_pid%p.log] " \ - "(%p replaced with pid)") \ - \ product(uintx, ShenandoahControlIntervalMin, 1, EXPERIMENTAL, \ "The minimum sleep interval for the control loop that drives " \ "the cycles. Lower values would increase GC responsiveness " \ From a1270d7edbedcd111755e2ed6da6260683d1eeaf Mon Sep 17 00:00:00 2001 From: William Kemper Date: Mon, 6 Mar 2023 17:52:07 +0000 Subject: [PATCH 190/254] Performance improvements for non-generational modes Reviewed-by: kdnilsen --- .../shenandoahAdaptiveHeuristics.cpp | 53 ++---- .../shenandoahAdaptiveHeuristics.hpp | 1 - .../heuristics/shenandoahHeuristics.cpp | 27 +-- .../heuristics/shenandoahHeuristics.hpp | 7 - .../gc/shenandoah/shenandoahCollectionSet.cpp | 9 +- .../gc/shenandoah/shenandoahCollectionSet.hpp | 4 +- .../gc/shenandoah/shenandoahConcurrentGC.cpp | 7 + .../share/gc/shenandoah/shenandoahFreeSet.cpp | 164 +++++++++++------- .../share/gc/shenandoah/shenandoahHeap.cpp | 3 + .../gc/shenandoah/shenandoahHeapRegion.cpp | 8 +- .../shenandoahHeapRegionCounters.cpp | 2 +- .../gc/shenandoah/shenandoah_globals.hpp | 4 +- 12 files changed, 147 insertions(+), 142 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp index 3fd9042ab8830..9145729ec3825 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp @@ -213,20 +213,27 @@ void ShenandoahAdaptiveHeuristics::choose_collection_set_from_regiondata(Shenand size_t free_target = (capacity * ShenandoahMinFreeThreshold) / 100 + max_cset; size_t min_garbage = (free_target > actual_free) ? (free_target - actual_free) : 0; - log_info(gc, ergo)("Adaptive CSet Selection. Max Evacuation: " SIZE_FORMAT "%s, Actual Free: " SIZE_FORMAT "%s.", - byte_size_in_proper_unit(max_cset), proper_unit_for_byte_size(max_cset), - byte_size_in_proper_unit(actual_free), proper_unit_for_byte_size(actual_free)); + log_info(gc, ergo)("Adaptive CSet Selection. Target Free: " SIZE_FORMAT "%s, Actual Free: " + SIZE_FORMAT "%s, Max Evacuation: " SIZE_FORMAT "%s, Min Garbage: " SIZE_FORMAT "%s", + byte_size_in_proper_unit(free_target), proper_unit_for_byte_size(free_target), + byte_size_in_proper_unit(actual_free), proper_unit_for_byte_size(actual_free), + byte_size_in_proper_unit(max_cset), proper_unit_for_byte_size(max_cset), + byte_size_in_proper_unit(min_garbage), proper_unit_for_byte_size(min_garbage)); size_t cur_cset = 0; size_t cur_garbage = 0; for (size_t idx = 0; idx < size; idx++) { ShenandoahHeapRegion* r = data[idx]._region; - size_t new_cset = cur_cset + r->get_live_data_bytes(); - size_t region_garbage = r->garbage(); - size_t new_garbage = cur_garbage + region_garbage; - bool add_regardless = (region_garbage > ignore_threshold) && (new_garbage < min_garbage); - if ((new_cset <= max_cset) && (add_regardless || (region_garbage > garbage_threshold))) { + + size_t new_cset = cur_cset + r->get_live_data_bytes(); + size_t new_garbage = cur_garbage + r->garbage(); + + if (new_cset > max_cset) { + break; + } + + if ((new_garbage < min_garbage) || (r->garbage() > garbage_threshold)) { cset->add_region(r); cur_cset = new_cset; cur_garbage = new_garbage; @@ -361,7 +368,7 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { size_t min_threshold = min_free_threshold(); - if (allocation_headroom < min_threshold) { + if (available < min_threshold) { log_info(gc)("Trigger (%s): Free (" SIZE_FORMAT "%s) is below minimum threshold (" SIZE_FORMAT "%s)", _generation->name(), byte_size_in_proper_unit(allocation_headroom), proper_unit_for_byte_size(allocation_headroom), @@ -373,7 +380,7 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { const size_t max_learn = ShenandoahLearningSteps; if (_gc_times_learned < max_learn) { size_t init_threshold = capacity / 100 * ShenandoahInitFreeThreshold; - if (allocation_headroom < init_threshold) { + if (available < init_threshold) { log_info(gc)("Trigger (%s): Learning " SIZE_FORMAT " of " SIZE_FORMAT ". Free (" SIZE_FORMAT "%s) is below initial threshold (" SIZE_FORMAT "%s)", _generation->name(), _gc_times_learned + 1, max_learn, byte_size_in_proper_unit(allocation_headroom), proper_unit_for_byte_size(allocation_headroom), @@ -423,33 +430,11 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { double avg_cycle_time = _gc_cycle_time_history->davg() + (_margin_of_error_sd * _gc_cycle_time_history->dsd()); - size_t last_live_memory = get_last_live_memory(); - size_t penultimate_live_memory = get_penultimate_live_memory(); - double original_cycle_time = avg_cycle_time; - if ((penultimate_live_memory < last_live_memory) && (penultimate_live_memory != 0)) { - // If the live-memory size is growing, our estimates of cycle time are based on lighter workload, so adjust. - // TODO: Be more precise about how to scale when live memory is growing. Existing code is a very rough approximation - // tuned with very limited workload observations. - avg_cycle_time = (avg_cycle_time * 2 * last_live_memory) / penultimate_live_memory; - } else { - int degen_cycles = degenerated_cycles_in_a_row(); - if (degen_cycles > 0) { - // If we've degenerated recently, we might be waiting too long between triggers so adjust trigger forward. - // TODO: Be more precise about how to scale when we've experienced recent degenerated GC. Existing code is a very - // rough approximation tuned with very limited workload observations. - avg_cycle_time += degen_cycles * avg_cycle_time; - } - } - double avg_alloc_rate = _allocation_rate.upper_bound(_margin_of_error_sd); log_debug(gc)("%s: average GC time: %.2f ms, allocation rate: %.0f %s/s", _generation->name(), avg_cycle_time * 1000, byte_size_in_proper_unit(avg_alloc_rate), proper_unit_for_byte_size(avg_alloc_rate)); if (avg_cycle_time > allocation_headroom / avg_alloc_rate) { - if (avg_cycle_time > original_cycle_time) { - log_debug(gc)("%s: average GC time adjusted from: %.2f ms to %.2f ms because upward trend in live memory retention", - _generation->name(), original_cycle_time, avg_cycle_time); - } log_info(gc)("Trigger (%s): Average GC time (%.2f ms) is above the time for average allocation rate (%.0f %sB/s) to deplete free headroom (" SIZE_FORMAT "%s) (margin of error = %.2f)", _generation->name(), avg_cycle_time * 1000, @@ -585,10 +570,6 @@ bool ShenandoahAllocationRate::is_spiking(double rate, double threshold) const { return false; } -double ShenandoahAllocationRate::instantaneous_rate(size_t allocated) const { - return instantaneous_rate(os::elapsedTime(), allocated); -} - double ShenandoahAllocationRate::instantaneous_rate(double time, size_t allocated) const { size_t last_value = _last_sample_value; double last_time = _last_sample_time; diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp index 6681909ca7077..d428babd36a70 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp @@ -36,7 +36,6 @@ class ShenandoahAllocationRate : public CHeapObj { double sample(size_t allocated); - double instantaneous_rate(size_t allocated) const; double upper_bound(double sds) const; bool is_spiking(double rate, double threshold) const; diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp index f8576e8cd66e6..e6cf7e19aa66d 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp @@ -58,8 +58,6 @@ ShenandoahHeuristics::ShenandoahHeuristics(ShenandoahGeneration* generation) : _gc_times_learned(0), _gc_time_penalties(0), _gc_cycle_time_history(new TruncatedSeq(Moving_Average_Samples, ShenandoahAdaptiveDecayFactor)), - _live_memory_last_cycle(0), - _live_memory_penultimate_cycle(0), _metaspace_oom() { // No unloading during concurrent mark? Communicate that to heuristics @@ -126,7 +124,7 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec for (size_t i = 0; i < num_regions; i++) { ShenandoahHeapRegion* region = heap->get_region(i); - if (!in_generation(region)) { + if (is_generational && !in_generation(region)) { continue; } @@ -181,8 +179,6 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec } } - save_last_live_memory(live_memory); - // Step 2. Look back at garbage statistics, and decide if we want to collect anything, // given the amount of immediately reclaimable garbage. If we do, figure out the collection set. @@ -217,8 +213,8 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec size_t collectable_garbage_percent = (total_garbage == 0) ? 0 : (collectable_garbage * 100 / total_garbage); log_info(gc, ergo)("Collectable Garbage: " SIZE_FORMAT "%s (" SIZE_FORMAT "%%), " - "Immediate: " SIZE_FORMAT "%s (" SIZE_FORMAT "%%), " - "CSet: " SIZE_FORMAT "%s (" SIZE_FORMAT "%%)", + "Immediate: " SIZE_FORMAT "%s (" SIZE_FORMAT "%%) R: " SIZE_FORMAT ", " + "CSet: " SIZE_FORMAT "%s (" SIZE_FORMAT "%%) R: " SIZE_FORMAT, byte_size_in_proper_unit(collectable_garbage), proper_unit_for_byte_size(collectable_garbage), @@ -227,10 +223,12 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec byte_size_in_proper_unit(immediate_garbage), proper_unit_for_byte_size(immediate_garbage), immediate_percent, + immediate_regions, byte_size_in_proper_unit(collection_set->garbage()), proper_unit_for_byte_size(collection_set->garbage()), - cset_percent); + cset_percent, + collection_set->count()); if (collection_set->garbage() > 0) { size_t young_evac_bytes = collection_set->get_young_bytes_reserved_for_evacuation(); @@ -382,16 +380,3 @@ size_t ShenandoahHeuristics::min_free_threshold() { : ShenandoahMinFreeThreshold; return _generation->soft_max_capacity() / 100 * min_free_threshold; } - -void ShenandoahHeuristics::save_last_live_memory(size_t live_memory) { - _live_memory_penultimate_cycle = _live_memory_last_cycle; - _live_memory_last_cycle = live_memory; -} - -size_t ShenandoahHeuristics::get_last_live_memory() { - return _live_memory_last_cycle; -} - -size_t ShenandoahHeuristics::get_penultimate_live_memory() { - return _live_memory_penultimate_cycle; -} diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp index 73aa18c19c598..31110867c455a 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp @@ -101,9 +101,6 @@ class ShenandoahHeuristics : public CHeapObj { intx _gc_time_penalties; TruncatedSeq* _gc_cycle_time_history; - size_t _live_memory_last_cycle; - size_t _live_memory_penultimate_cycle; - // There may be many threads that contend to set this flag ShenandoahSharedFlag _metaspace_oom; @@ -173,10 +170,6 @@ class ShenandoahHeuristics : public CHeapObj { virtual void initialize(); double elapsed_cycle_time() const; - - void save_last_live_memory(size_t live_memory); - size_t get_last_live_memory(); - size_t get_penultimate_live_memory(); }; #endif // SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHHEURISTICS_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp index 1b8aefadeeebf..7acf8d7ce2112 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp @@ -43,6 +43,7 @@ ShenandoahCollectionSet::ShenandoahCollectionSet(ShenandoahHeap* heap, ReservedS _has_old_regions(false), _garbage(0), _used(0), + _live(0), _region_count(0), _old_garbage(0), _current_index(0) { @@ -105,6 +106,7 @@ void ShenandoahCollectionSet::add_region(ShenandoahHeapRegion* r) { _has_old_regions |= r->is_old(); _garbage += r->garbage(); _used += r->used(); + _live += r->get_live_data_bytes(); // Update the region status too. State transition would be checked internally. r->make_cset(); } @@ -122,6 +124,7 @@ void ShenandoahCollectionSet::clear() { _garbage = 0; _old_garbage = 0; _used = 0; + _live = 0; _region_count = 0; _current_index = 0; @@ -177,7 +180,11 @@ ShenandoahHeapRegion* ShenandoahCollectionSet::next() { } void ShenandoahCollectionSet::print_on(outputStream* out) const { - out->print_cr("Collection Set : " SIZE_FORMAT "", count()); + out->print_cr("Collection Set: Regions: " + SIZE_FORMAT ", Garbage: " SIZE_FORMAT "%s, Live: " SIZE_FORMAT "%s, Used: " SIZE_FORMAT "%s", count(), + byte_size_in_proper_unit(garbage()), proper_unit_for_byte_size(garbage()), + byte_size_in_proper_unit(live()), proper_unit_for_byte_size(live()), + byte_size_in_proper_unit(used()), proper_unit_for_byte_size(used())); debug_only(size_t regions = 0;) for (size_t index = 0; index < _heap->num_regions(); index ++) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp index 6dd5a29dfe3c0..163e2f1cca463 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp @@ -46,6 +46,7 @@ class ShenandoahCollectionSet : public CHeapObj { bool _has_old_regions; size_t _garbage; size_t _used; + size_t _live; size_t _region_count; size_t _immediate_trash; @@ -119,8 +120,9 @@ class ShenandoahCollectionSet : public CHeapObj { bool has_old_regions() const { return _has_old_regions; } size_t used() const { return _used; } - + size_t live() const { return _live; } size_t garbage() const { return _garbage; } + void clear(); private: diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index 7a24d1696f445..98c758a0bf5a6 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -747,6 +747,13 @@ void ShenandoahConcurrentGC::op_final_mark() { heap->prepare_concurrent_roots(); if (!heap->collection_set()->is_empty()) { + LogTarget(Info, gc, ergo) lt; + if (lt.is_enabled()) { + ResourceMark rm; + LogStream ls(lt); + heap->collection_set()->print_on(&ls); + } + if (ShenandoahVerify) { heap->verifier()->verify_before_evacuation(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index b3dee45c93ad6..6389c329ce928 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -106,6 +106,7 @@ HeapWord* ShenandoahFreeSet::allocate_with_affiliation(ShenandoahRegionAffiliati } } } + log_debug(gc, free)("Could not allocate collector region with affiliation: %s for request " PTR_FORMAT, affiliation_name(affiliation), p2i(&req)); return nullptr; } @@ -125,25 +126,27 @@ HeapWord* ShenandoahFreeSet::allocate_single(ShenandoahAllocRequest& req, bool& // Overwrite with non-zero (non-NULL) values only if necessary for allocation bookkeeping. bool allow_new_region = true; - switch (req.affiliation()) { - case ShenandoahRegionAffiliation::OLD_GENERATION: - // Note: unsigned result from adjusted_unaffiliated_regions() will never be less than zero, but it may equal zero. - if (_heap->old_generation()->adjusted_unaffiliated_regions() <= 0) { - allow_new_region = false; - } - break; + if (_heap->mode()->is_generational()) { + switch (req.affiliation()) { + case ShenandoahRegionAffiliation::OLD_GENERATION: + // Note: unsigned result from adjusted_unaffiliated_regions() will never be less than zero, but it may equal zero. + if (_heap->old_generation()->adjusted_unaffiliated_regions() <= 0) { + allow_new_region = false; + } + break; - case ShenandoahRegionAffiliation::YOUNG_GENERATION: - // Note: unsigned result from adjusted_unaffiliated_regions() will never be less than zero, but it may equal zero. - if (_heap->young_generation()->adjusted_unaffiliated_regions() <= 0) { - allow_new_region = false; - } - break; + case ShenandoahRegionAffiliation::YOUNG_GENERATION: + // Note: unsigned result from adjusted_unaffiliated_regions() will never be less than zero, but it may equal zero. + if (_heap->young_generation()->adjusted_unaffiliated_regions() <= 0) { + allow_new_region = false; + } + break; - case ShenandoahRegionAffiliation::FREE: - default: - ShouldNotReachHere(); - break; + case ShenandoahRegionAffiliation::FREE: + default: + ShouldNotReachHere(); + break; + } } switch (req.type()) { @@ -171,24 +174,38 @@ HeapWord* ShenandoahFreeSet::allocate_single(ShenandoahAllocRequest& req, bool& // PLABs always reside in old-gen and are only allocated during evacuation phase. case ShenandoahAllocRequest::_alloc_shared_gc: { - // First try to fit into a region that is already in use in the same generation. - HeapWord* result; - if (req.affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) { - // TODO: this is a work around to address a deficiency in FreeSet representation. A better solution fixes - // the FreeSet implementation to deal more efficiently with old-gen regions as being in the "collector free set" - result = allocate_with_old_affiliation(req, in_new_region); + if (!_heap->mode()->is_generational()) { + // size_t is unsigned, need to dodge underflow when _leftmost = 0 + // Fast-path: try to allocate in the collector view first + for (size_t c = _collector_rightmost + 1; c > _collector_leftmost; c--) { + size_t idx = c - 1; + if (is_collector_free(idx)) { + HeapWord* result = try_allocate_in(_heap->get_region(idx), req, in_new_region); + if (result != nullptr) { + return result; + } + } + } } else { - result = allocate_with_affiliation(req.affiliation(), req, in_new_region); - } - if (result != nullptr) { - return result; - } - if (allow_new_region) { - // Then try a free region that is dedicated to GC allocations. - result = allocate_with_affiliation(FREE, req, in_new_region); + // First try to fit into a region that is already in use in the same generation. + HeapWord* result; + if (req.affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) { + // TODO: this is a work around to address a deficiency in FreeSet representation. A better solution fixes + // the FreeSet implementation to deal more efficiently with old-gen regions as being in the "collector free set" + result = allocate_with_old_affiliation(req, in_new_region); + } else { + result = allocate_with_affiliation(req.affiliation(), req, in_new_region); + } if (result != nullptr) { return result; } + if (allow_new_region) { + // Then try a free region that is dedicated to GC allocations. + result = allocate_with_affiliation(FREE, req, in_new_region); + if (result != nullptr) { + return result; + } + } } // No dice. Can we borrow space from mutator view? @@ -206,6 +223,7 @@ HeapWord* ShenandoahFreeSet::allocate_single(ShenandoahAllocRequest& req, bool& flip_to_gc(r); HeapWord *result = try_allocate_in(r, req, in_new_region); if (result != nullptr) { + log_debug(gc, free)("Flipped region " SIZE_FORMAT " to gc for request: " PTR_FORMAT, idx, p2i(&req)); return result; } } @@ -248,13 +266,8 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah assert(ctx->top_at_mark_start(r) == r->bottom(), "Newly established allocation region starts with TAMS equal to bottom"); assert(ctx->is_bitmap_clear_range(ctx->top_bitmap(r), r->end()), "Bitmap above top_bitmap() must be clear"); - - // Leave top_bitmap alone. The first time a heap region is put into service, top_bitmap should equal end. - // Thereafter, it should represent the upper bound on parts of the bitmap that need to be cleared. - log_debug(gc)("NOT clearing bitmap for region " SIZE_FORMAT ", top_bitmap: " - PTR_FORMAT " at transition from FREE to %s", - r->index(), p2i(ctx->top_bitmap(r)), affiliation_name(req.affiliation())); } else if (r->affiliation() != req.affiliation()) { + assert(!_heap->mode()->is_generational(), "Should not have conflicting affiliation in non-generational mode."); return nullptr; } @@ -262,9 +275,15 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah HeapWord* result = nullptr; size_t size = req.size(); + if (in_new_region) { + log_debug(gc, free)("Using new region (" SIZE_FORMAT ") for %s (" PTR_FORMAT ").", + r->index(), ShenandoahAllocRequest::alloc_type_to_string(req.type()), p2i(&req)); + } + // req.size() is in words, r->free() is in bytes. if (ShenandoahElasticTLAB && req.is_lab_alloc()) { if (req.type() == ShenandoahAllocRequest::_alloc_plab) { + assert(_heap->mode()->is_generational(), "PLABs are only for generational mode"); // Need to assure that plabs are aligned on multiple of card region. size_t free = r->free(); size_t usable_free = (free / CardTable::card_size()) << CardTable::card_shift(); @@ -329,11 +348,12 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah } assert (result != nullptr, "Allocation must succeed: free " SIZE_FORMAT ", actual " SIZE_FORMAT, free, size); } else { - log_trace(gc, ergo)("Failed to shrink TLAB or GCLAB request (" SIZE_FORMAT ") in region " SIZE_FORMAT " to " SIZE_FORMAT + log_trace(gc, free)("Failed to shrink TLAB or GCLAB request (" SIZE_FORMAT ") in region " SIZE_FORMAT " to " SIZE_FORMAT " because min_size() is " SIZE_FORMAT, req.size(), r->index(), size, req.min_size()); } } } else if (req.is_lab_alloc() && req.type() == ShenandoahAllocRequest::_alloc_plab) { + assert(_heap->mode()->is_generational(), "PLABs are only for generational mode"); // inelastic PLAB size_t free = r->free(); size_t usable_free = (free / CardTable::card_size()) << CardTable::card_shift(); @@ -373,11 +393,12 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah } } + ShenandoahGeneration* generation = _heap->generation_for(req.affiliation()); if (result != nullptr) { // Allocation successful, bump stats: if (req.is_mutator_alloc()) { - // Mutator allocations always pull from young gen. - _heap->young_generation()->increase_used(size * HeapWordSize); + assert(req.is_young(), "Mutator allocations always come from young generation."); + generation->increase_used(size * HeapWordSize); increase_used(size * HeapWordSize); } else { assert(req.is_gc_alloc(), "Should be gc_alloc since req wasn't mutator alloc"); @@ -390,17 +411,14 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah // PLABs parsable while still allowing the PLAB to serve future allocation requests that arise during the // next evacuation pass. r->set_update_watermark(r->top()); - - if (r->affiliation() == ShenandoahRegionAffiliation::YOUNG_GENERATION) { - _heap->young_generation()->increase_used(size * HeapWordSize); - } else { - assert(r->affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION, "GC Alloc was not YOUNG so must be OLD"); + generation->increase_used(size * HeapWordSize); + if (r->affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) { assert(req.type() != ShenandoahAllocRequest::_alloc_gclab, "old-gen allocations use PLAB or shared allocation"); - _heap->old_generation()->increase_used(size * HeapWordSize); // for plabs, we'll sort the difference between evac and promotion usage when we retire the plab } } } + if (result == nullptr || has_no_alloc_capacity(r)) { // Region cannot afford this or future allocations. Retire it. // @@ -415,7 +433,7 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah size_t waste = r->free(); if (waste > 0) { increase_used(waste); - _heap->generation_for(req.affiliation())->increase_allocated(waste); + generation->increase_allocated(waste); _heap->notify_mutator_alloc_words(waste >> LogHeapWordSize, true); } } @@ -471,11 +489,18 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req) { size_t num = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize); assert(req.affiliation() == ShenandoahRegionAffiliation::YOUNG_GENERATION, "Humongous regions always allocated in YOUNG"); - size_t avail_young_regions = _heap->young_generation()->adjusted_unaffiliated_regions(); + ShenandoahGeneration* generation = _heap->generation_for(req.affiliation()); - // No regions left to satisfy allocation, bye. - if (num > mutator_count() || (num > avail_young_regions)) { - return nullptr; + // Check if there are enough regions left to satisfy allocation. + if (_heap->mode()->is_generational()) { + size_t avail_young_regions = generation->adjusted_unaffiliated_regions(); + if (num > mutator_count() || (num > avail_young_regions)) { + return nullptr; + } + } else { + if (num > mutator_count()) { + return nullptr; + } } // Find the continuous interval of $num regions, starting from $beg and ending in $end, @@ -542,9 +567,9 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req) { // Leave top_bitmap alone. The first time a heap region is put into service, top_bitmap should equal end. // Thereafter, it should represent the upper bound on parts of the bitmap that need to be cleared. // ctx->clear_bitmap(r); - log_debug(gc)("NOT clearing bitmap for Humongous region [" PTR_FORMAT ", " PTR_FORMAT "], top_bitmap: " - PTR_FORMAT " at transition from FREE to %s", - p2i(r->bottom()), p2i(r->end()), p2i(ctx->top_bitmap(r)), affiliation_name(req.affiliation())); + log_debug(gc, free)("NOT clearing bitmap for Humongous region [" PTR_FORMAT ", " PTR_FORMAT "], top_bitmap: " + PTR_FORMAT " at transition from FREE to %s", + p2i(r->bottom()), p2i(r->end()), p2i(ctx->top_bitmap(r)), affiliation_name(req.affiliation())); _mutator_free_bitmap.clear_bit(r->index()); } @@ -552,17 +577,13 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req) { // While individual regions report their true use, all humongous regions are // marked used in the free set. increase_used(ShenandoahHeapRegion::region_size_bytes() * num); - if (req.affiliation() == ShenandoahRegionAffiliation::YOUNG_GENERATION) { - _heap->young_generation()->increase_used(words_size * HeapWordSize); - } else if (req.affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) { - _heap->old_generation()->increase_used(words_size * HeapWordSize); - } + generation->increase_used(words_size * HeapWordSize); if (remainder != 0) { // Record this remainder as allocation waste size_t waste = ShenandoahHeapRegion::region_size_words() - remainder; _heap->notify_mutator_alloc_words(waste, true); - _heap->generation_for(req.affiliation())->increase_allocated(waste * HeapWordSize); + generation->increase_allocated(waste * HeapWordSize); } // Allocated at left/rightmost? Move the bounds appropriately. @@ -570,6 +591,7 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req) { adjust_bounds(); } assert_bounds(); + req.set_actual_size(words_size); return _heap->get_region(beg)->bottom(); } @@ -601,6 +623,7 @@ void ShenandoahFreeSet::try_recycle_trashed(ShenandoahHeapRegion *r) { void ShenandoahFreeSet::recycle_trash() { // lock is not reentrable, check we don't have it shenandoah_assert_not_heaplocked(); + for (size_t i = 0; i < _heap->num_regions(); i++) { ShenandoahHeapRegion* r = _heap->get_region(i); if (r->is_trash()) { @@ -654,7 +677,7 @@ void ShenandoahFreeSet::rebuild() { shenandoah_assert_heaplocked(); clear(); - log_debug(gc)("Rebuilding FreeSet"); + log_debug(gc, free)("Rebuilding FreeSet"); for (size_t idx = 0; idx < _heap->num_regions(); idx++) { ShenandoahHeapRegion* region = _heap->get_region(idx); if (region->is_alloc_allowed() || region->is_trash()) { @@ -669,7 +692,9 @@ void ShenandoahFreeSet::rebuild() { assert(!is_mutator_free(idx), "We are about to add it, it shouldn't be there already"); _mutator_free_bitmap.set_bit(idx); - log_debug(gc)(" Setting Region " SIZE_FORMAT " _mutator_free_bitmap bit to true", idx); + log_debug(gc, free)(" Adding Region " SIZE_FORMAT " (Free: " SIZE_FORMAT "%s, Used: " SIZE_FORMAT "%s) to mutator free set", + idx, byte_size_in_proper_unit(region->free()), proper_unit_for_byte_size(region->free()), + byte_size_in_proper_unit(region->used()), proper_unit_for_byte_size(region->used())); } } @@ -706,7 +731,9 @@ void ShenandoahFreeSet::reserve_regions(size_t to_reserve) { size_t ac = alloc_capacity(region); _capacity -= ac; reserved += ac; - log_debug(gc)(" Shifting region " SIZE_FORMAT " from mutator_free to collector_free", idx); + log_debug(gc, free)(" Shifting Region " SIZE_FORMAT " (Free: " SIZE_FORMAT "%s, Used: " SIZE_FORMAT "%s) to collector free set", + idx, byte_size_in_proper_unit(region->free()), proper_unit_for_byte_size(region->free()), + byte_size_in_proper_unit(region->used()), proper_unit_for_byte_size(region->used())); } } } @@ -714,7 +741,7 @@ void ShenandoahFreeSet::reserve_regions(size_t to_reserve) { void ShenandoahFreeSet::log_status() { shenandoah_assert_heaplocked(); - LogTarget(Info, gc, ergo) lt; + LogTarget(Info, gc, free) lt; if (lt.is_enabled()) { ResourceMark rm; LogStream ls(lt); @@ -780,11 +807,14 @@ void ShenandoahFreeSet::log_status() { frag_int = 0; } ls.print(SIZE_FORMAT "%% internal; ", frag_int); + ls.print("Used: " SIZE_FORMAT "%s, Mutator Free: " SIZE_FORMAT " ", + byte_size_in_proper_unit(total_used), proper_unit_for_byte_size(total_used), mutator_count()); } { size_t max = 0; size_t total_free = 0; + size_t total_used = 0; for (size_t idx = _collector_leftmost; idx <= _collector_rightmost; idx++) { if (is_collector_free(idx)) { @@ -792,12 +822,14 @@ void ShenandoahFreeSet::log_status() { size_t free = alloc_capacity(r); max = MAX2(max, free); total_free += free; + total_used += r->used(); } } - ls.print_cr("Reserve: " SIZE_FORMAT "%s, Max: " SIZE_FORMAT "%s", + ls.print_cr("Reserve: " SIZE_FORMAT "%s, Max: " SIZE_FORMAT "%s, Used: " SIZE_FORMAT "%s", byte_size_in_proper_unit(total_free), proper_unit_for_byte_size(total_free), - byte_size_in_proper_unit(max), proper_unit_for_byte_size(max)); + byte_size_in_proper_unit(max), proper_unit_for_byte_size(max), + byte_size_in_proper_unit(total_used), proper_unit_for_byte_size(total_used)); } } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 35798fdc10167..802674b1bce50 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -879,8 +879,10 @@ HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) // Limit growth of GCLABs to ShenandoahMaxEvacLABRatio * the minimum size. This enables more equitable distribution of // available evacuation buidget between the many threads that are coordinating in the evacuation effort. if (ShenandoahMaxEvacLABRatio > 0) { + log_debug(gc, free)("Allocate new gclab: " SIZE_FORMAT ", " SIZE_FORMAT, new_size, PLAB::min_size() * ShenandoahMaxEvacLABRatio); new_size = MIN2(new_size, PLAB::min_size() * ShenandoahMaxEvacLABRatio); } + new_size = MIN2(new_size, PLAB::max_size()); new_size = MAX2(new_size, PLAB::min_size()); @@ -892,6 +894,7 @@ HeapWord* ShenandoahHeap::allocate_from_gclab_slow(Thread* thread, size_t size) if (new_size < size) { // New size still does not fit the object. Fall back to shared allocation. // This avoids retiring perfectly good GCLABs, when we encounter a large object. + log_debug(gc, free)("New gclab size (" SIZE_FORMAT ") is too small for " SIZE_FORMAT, new_size, size); return nullptr; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index fca5ce954980c..5efe67e247015 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -435,7 +435,7 @@ void ShenandoahHeapRegion::print_on(outputStream* st) const { st->print("|T " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_tlab_allocs()), proper_unit_for_byte_size(get_tlab_allocs())); st->print("|G " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_gclab_allocs()), proper_unit_for_byte_size(get_gclab_allocs())); if (ShenandoahHeap::heap()->mode()->is_generational()) { - st->print("|G " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_plab_allocs()), proper_unit_for_byte_size(get_plab_allocs())); + st->print("|P " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_plab_allocs()), proper_unit_for_byte_size(get_plab_allocs())); } st->print("|S " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_shared_allocs()), proper_unit_for_byte_size(get_shared_allocs())); st->print("|L " SIZE_FORMAT_W(5) "%1s", byte_size_in_proper_unit(get_live_data_bytes()), proper_unit_for_byte_size(get_live_data_bytes())); @@ -666,11 +666,7 @@ void ShenandoahHeapRegion::recycle() { ShenandoahHeap* heap = ShenandoahHeap::heap(); shenandoah_assert_heaplocked(); - if (affiliation() == YOUNG_GENERATION) { - heap->young_generation()->decrease_used(used()); - } else if (affiliation() == OLD_GENERATION) { - heap->old_generation()->decrease_used(used()); - } + heap->generation_for(affiliation())->decrease_used(used()); set_top(bottom()); clear_live_data(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp index 209ca605d88ff..b10816d7e98b4 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp @@ -86,7 +86,7 @@ void ShenandoahHeapRegionCounters::write_snapshot(PerfLongVariable** regions, PerfLongVariable* status, size_t num_regions, size_t region_size, size_t protocol_version) { - LogTarget(Debug, gc, region) lt; + LogTarget(Trace, gc, region) lt; if (lt.is_enabled()) { ResourceMark rm; LogStream ls(lt); diff --git a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp index a664353e97921..cb740f8e16d9b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp @@ -311,7 +311,7 @@ "which upgrades to full GC whenever the budget is exceeded.") \ range(1.0,100.0) \ \ - product(uintx, ShenandoahMaxEvacLABRatio, 16, EXPERIMENTAL, \ + product(uintx, ShenandoahMaxEvacLABRatio, 0, EXPERIMENTAL, \ "Potentially, each running thread maintains a PLAB for " \ "evacuating objects into old-gen memory and a GCLAB for " \ "evacuating objects into young-gen memory. Each time a thread " \ @@ -327,7 +327,7 @@ "stop-the-world pauses. This is because a large value " \ "allows individual threads to consume large percentages of " \ "the total evacuation budget without necessarily effectively " \ - "filling their local evcauation buffers with evacuated " \ + "filling their local evacuation buffers with evacuated " \ "objects. A value of zero means no maximum size is enforced.") \ range(0, 1024) \ \ From 87bf68f35c05c063c8a5865b22e1b9704f99e7e5 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Fri, 10 Mar 2023 00:46:29 +0000 Subject: [PATCH 191/254] Use relevant variables in log messages, remove unused import Reviewed-by: ysr, kdnilsen --- .../shenandoahAdaptiveHeuristics.cpp | 37 ++++++------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp index 9145729ec3825..c61e53299f2fc 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp @@ -28,7 +28,6 @@ #include "gc/shenandoah/shenandoahCollectionSet.hpp" #include "gc/shenandoah/shenandoahFreeSet.hpp" #include "gc/shenandoah/shenandoahGeneration.hpp" -#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" #include "gc/shenandoah/shenandoahYoungGeneration.hpp" #include "logging/log.hpp" #include "logging/logTag.hpp" @@ -340,28 +339,6 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { available = usable; } - // Allocation spikes are a characteristic of both the application ahd the JVM configuration. On the JVM command line, - // the application developer may want to supply a hint of the nature of spikes that are inherent in the application - // workload, and this information would normally be independent of heap size (not a percentage thereof). On the - // other hand, some allocation spikes are correlated with JVM configuration. For example, there are allocation - // spikes at the starts of concurrent marking and evacuation to refresh all local allocation buffers. The nature - // of these spikes is determined by LAB min and max sizes and numbers of threads, but also on frequency of GC passes, - // and on "periodic" behavior of these threads If GC frequency is much higher than the periodic trigger for mutator - // threads, then many of the mutator threads may be able to "sit out" of most GC passes. Though the thread's stack - // must be scanned, the thread does not need to refresh its LABs if it sits idle throughout the duration of the GC - // pass. The best prediction for this aspect of spikes in allocation patterns is probably recent past history. - // TODO: and dive deeper into _gc_time_penalties as this may also need to be corrected - - // Check if allocation headroom is still okay. This also factors in: - // 1. Some space to absorb allocation spikes (ShenandoahAllocSpikeFactor) - // 2. Accumulated penalties from Degenerated and Full GC - size_t allocation_headroom = available; - size_t spike_headroom = capacity / 100 * ShenandoahAllocSpikeFactor; - size_t penalties = capacity / 100 * _gc_time_penalties; - - allocation_headroom -= MIN2(allocation_headroom, penalties); - allocation_headroom -= MIN2(allocation_headroom, spike_headroom); - // Track allocation rate even if we decide to start a cycle for other reasons. double rate = _allocation_rate.sample(allocated); _last_trigger = OTHER; @@ -371,7 +348,7 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { if (available < min_threshold) { log_info(gc)("Trigger (%s): Free (" SIZE_FORMAT "%s) is below minimum threshold (" SIZE_FORMAT "%s)", _generation->name(), - byte_size_in_proper_unit(allocation_headroom), proper_unit_for_byte_size(allocation_headroom), + byte_size_in_proper_unit(available), proper_unit_for_byte_size(available), byte_size_in_proper_unit(min_threshold), proper_unit_for_byte_size(min_threshold)); return resize_and_evaluate(); } @@ -383,7 +360,7 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { if (available < init_threshold) { log_info(gc)("Trigger (%s): Learning " SIZE_FORMAT " of " SIZE_FORMAT ". Free (" SIZE_FORMAT "%s) is below initial threshold (" SIZE_FORMAT "%s)", _generation->name(), _gc_times_learned + 1, max_learn, - byte_size_in_proper_unit(allocation_headroom), proper_unit_for_byte_size(allocation_headroom), + byte_size_in_proper_unit(available), proper_unit_for_byte_size(available), byte_size_in_proper_unit(init_threshold), proper_unit_for_byte_size(init_threshold)); return true; } @@ -428,6 +405,16 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { // ShenandoahControlIntervalMax. The current control interval (or the max control interval) should also be added into // the calculation of avg_cycle_time below. + // Check if allocation headroom is still okay. This also factors in: + // 1. Some space to absorb allocation spikes (ShenandoahAllocSpikeFactor) + // 2. Accumulated penalties from Degenerated and Full GC + size_t allocation_headroom = available; + size_t spike_headroom = capacity / 100 * ShenandoahAllocSpikeFactor; + size_t penalties = capacity / 100 * _gc_time_penalties; + + allocation_headroom -= MIN2(allocation_headroom, penalties); + allocation_headroom -= MIN2(allocation_headroom, spike_headroom); + double avg_cycle_time = _gc_cycle_time_history->davg() + (_margin_of_error_sd * _gc_cycle_time_history->dsd()); double avg_alloc_rate = _allocation_rate.upper_bound(_margin_of_error_sd); From b4ee731658e100c8375099cbcb9496b6747c1d7e Mon Sep 17 00:00:00 2001 From: William Kemper Date: Mon, 13 Mar 2023 19:00:54 +0000 Subject: [PATCH 192/254] Fix incorrect assert logic Reviewed-by: kdnilsen --- src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index 6389c329ce928..32cc166964018 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -267,7 +267,8 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah assert(ctx->top_at_mark_start(r) == r->bottom(), "Newly established allocation region starts with TAMS equal to bottom"); assert(ctx->is_bitmap_clear_range(ctx->top_bitmap(r), r->end()), "Bitmap above top_bitmap() must be clear"); } else if (r->affiliation() != req.affiliation()) { - assert(!_heap->mode()->is_generational(), "Should not have conflicting affiliation in non-generational mode."); + assert(_heap->mode()->is_generational(), "Request for %s from %s region should only happen in generational mode.", + affiliation_name(req.affiliation()), affiliation_name(r->affiliation())); return nullptr; } From bfc07d8ab767a9b3df16fa2813034ee915fc6202 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Fri, 17 Mar 2023 00:11:39 +0000 Subject: [PATCH 193/254] Fix off-by-one error when handling pinned regions in a mixed collection Reviewed-by: kdnilsen, ysr --- .../heuristics/shenandoahOldHeuristics.cpp | 69 +++++++-------- .../heuristics/shenandoahOldHeuristics.hpp | 4 - .../test_shenandoahOldHeuristic.cpp | 85 +++++++++++++------ 3 files changed, 89 insertions(+), 69 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp index e7a8b304445cb..32d2f02f39c6e 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp @@ -36,9 +36,6 @@ uint ShenandoahOldHeuristics::NOT_FOUND = -1U; ShenandoahOldHeuristics::ShenandoahOldHeuristics(ShenandoahOldGeneration* generation, ShenandoahHeuristics* trigger_heuristic) : ShenandoahHeuristics(generation), -#ifdef ASSERT - _start_candidate(0), -#endif _first_pinned_candidate(NOT_FOUND), _last_old_collection_candidate(0), _next_old_collection_candidate(0), @@ -122,12 +119,15 @@ bool ShenandoahOldHeuristics::prime_collection_set(ShenandoahCollectionSet* coll } void ShenandoahOldHeuristics::slide_pinned_regions_to_front() { - // Find the leftmost unpinned region. The region in this slot will have been - // added to the cset, so we can use it to hold pointers to regions that were - // pinned when the cset was chosen. - // [ r p r p p p r ] - // ^ - // | first r to the left should be in the collection set now. + // Find the first unpinned region to the left of the next region that + // will be added to the collection set. These regions will have been + // added to the cset, so we can use them to hold pointers to regions + // that were pinned when the cset was chosen. + // [ r p r p p p r r ] + // ^ ^ ^ + // | | | pointer to next region to add to a mixed collection is here. + // | | first r to the left should be in the collection set now. + // | first pinned region, we don't need to look past this uint write_index = NOT_FOUND; for (uint search = _next_old_collection_candidate - 1; search > _first_pinned_candidate; --search) { ShenandoahHeapRegion* region = _region_data[search]._region; @@ -138,45 +138,40 @@ void ShenandoahOldHeuristics::slide_pinned_regions_to_front() { } } + // If we could not find an unpinned region, it means there are no slots available + // to move up the pinned regions. In this case, we just reset our next index in the + // hopes that some of these regions will become unpinned before the next mixed + // collection. We may want to bailout of here instead, as it should be quite + // rare to have so many pinned regions and may indicate something is wrong. if (write_index == NOT_FOUND) { - if (_first_pinned_candidate > 0) { - _next_old_collection_candidate = _first_pinned_candidate; - } + assert(_first_pinned_candidate != NOT_FOUND, "Should only be here if there are pinned regions."); + _next_old_collection_candidate = _first_pinned_candidate; return; } // Find pinned regions to the left and move their pointer into a slot - // that was pointing at a region that has been added to the cset. - // [ r p r p p p r ] - // ^ - // | Write pointer is here. We know this region is already in the cset - // | so we can clobber it with the next pinned region we find. - for (size_t search = write_index - 1; search > _first_pinned_candidate; --search) { + // that was pointing at a region that has been added to the cset (or was pointing + // to a pinned region that we've already moved up). We are done when the leftmost + // pinned region has been slid up. + // [ r p r x p p p r ] + // ^ ^ + // | | next region for mixed collections + // | Write pointer is here. We know this region is already in the cset + // | so we can clobber it with the next pinned region we find. + for (int32_t search = write_index - 1; search >= (int32_t)_first_pinned_candidate; --search) { RegionData& skipped = _region_data[search]; if (skipped._region->is_pinned()) { - RegionData& added_to_cset = _region_data[write_index]; - assert(added_to_cset._region->is_cset(), "Can only overwrite slots used by regions added to the collection set."); - added_to_cset._region = skipped._region; - added_to_cset._garbage = skipped._garbage; + RegionData& available_slot = _region_data[write_index]; + available_slot._region = skipped._region; + available_slot._garbage = skipped._garbage; --write_index; } } - // Everything left should already be in the cset - // [ r x p p p p r ] - // ^ - // | next pointer points at the first region which was not added - // | to the collection set. -#ifdef ASSERT - for (size_t check = write_index - 1; check > _start_candidate; --check) { - ShenandoahHeapRegion* region = _region_data[check]._region; - assert(region->is_cset(), "All regions here should be in the collection set."); - } - _start_candidate = write_index; -#endif - - // Update to read from the leftmost pinned region. - _next_old_collection_candidate = write_index; + // Update to read from the leftmost pinned region. Plus one here because we decremented + // the write index to hold the next found pinned region. We are just moving it back now + // to point to the first pinned region. + _next_old_collection_candidate = write_index + 1; } // Both arguments are don't cares for old-gen collections diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp index 3fc5c7f2ff2b9..09a49977e0f61 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp @@ -48,10 +48,6 @@ class ShenandoahOldHeuristics : public ShenandoahHeuristics { // have yet to be added to a mixed collection. There is also some special // handling for pinned regions, described further below. - // This points to the first candidate of the current mixed collection. This - // is only used for an assertion when handling pinned regions. - debug_only(uint _start_candidate); - // Pinned regions may not be included in the collection set. Any old regions // which were pinned at the time when old regions were added to the mixed // collection will have been skipped. These regions are still contain garbage, diff --git a/test/hotspot/gtest/gc/shenandoah/test_shenandoahOldHeuristic.cpp b/test/hotspot/gtest/gc/shenandoah/test_shenandoahOldHeuristic.cpp index eca43480b9f9a..5307ed9584e0b 100644 --- a/test/hotspot/gtest/gc/shenandoah/test_shenandoahOldHeuristic.cpp +++ b/test/hotspot/gtest/gc/shenandoah/test_shenandoahOldHeuristic.cpp @@ -5,12 +5,13 @@ #include "gc/shenandoah/shenandoahGeneration.hpp" #include "gc/shenandoah/shenandoahOldGeneration.hpp" #include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp" +#include // These tests will all be skipped (unless Shenandoah becomes the default // collector). To execute these tests, you must enable Shenandoah, which // is done with: // -// % _JAVA_OPTIONS="-XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational" make exploded-test TEST="gtest:Shenandoah*" +// % make exploded-test TEST="gtest:ShenandoahOld*" CONF=release TEST_OPTS="JAVA_OPTIONS=-XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational" // // Please note that these 'unit' tests are really integration tests and rely // on the JVM being initialized. These tests manipulate the state of the @@ -110,6 +111,26 @@ class ShenandoahOldHeuristicTest : public ::testing::Test { size_t collection_threshold() const { return ShenandoahHeapRegion::region_size_bytes() * ShenandoahOldGarbageThreshold / 100; } + + bool collection_set_is(size_t r1) { return _collection_set_is(1, r1); } + bool collection_set_is(size_t r1, size_t r2) { return _collection_set_is(2, r1, r2); } + bool collection_set_is(size_t r1, size_t r2, size_t r3) { return _collection_set_is(3, r1, r2, r3); } + + bool _collection_set_is(size_t count, ...) { + va_list args; + va_start(args, count); + EXPECT_EQ(count, _collection_set->count()); + bool result = true; + for (size_t i = 0; i < count; ++i) { + size_t index = va_arg(args, size_t); + if (!_collection_set->is_in(index)) { + result = false; + break; + } + } + va_end(args); + return result; + } }; TEST_VM_F(ShenandoahOldHeuristicTest, select_no_old_regions) { @@ -150,6 +171,7 @@ TEST_VM_F(ShenandoahOldHeuristicTest, prime_one_old_region) { _heuristics->prepare_for_old_collections(); _heuristics->prime_collection_set(_collection_set); + EXPECT_TRUE(collection_set_is(10UL)); EXPECT_EQ(garbage, _collection_set->get_old_garbage()); EXPECT_EQ(0U, _heuristics->unprocessed_old_collection_candidates()); } @@ -162,6 +184,7 @@ TEST_VM_F(ShenandoahOldHeuristicTest, prime_many_old_regions) { _heuristics->prepare_for_old_collections(); _heuristics->prime_collection_set(_collection_set); + EXPECT_TRUE(collection_set_is(100UL, 101UL)); EXPECT_EQ(g1 + g2, _collection_set->get_old_garbage()); EXPECT_EQ(0U, _heuristics->unprocessed_old_collection_candidates()); } @@ -181,15 +204,15 @@ TEST_VM_F(ShenandoahOldHeuristicTest, skip_pinned_regions) { SKIP_IF_NOT_SHENANDOAH(); // Create three old regions with enough garbage to be collected. - size_t g1 = make_garbage_above_threshold(1); - size_t g2 = make_garbage_above_threshold(2); - size_t g3 = make_garbage_above_threshold(3); + size_t g1 = make_garbage_above_threshold(0); + size_t g2 = make_garbage_above_threshold(1); + size_t g3 = make_garbage_above_threshold(2); // A region can be pinned when we chose collection set candidates. - make_pinned(2); + make_pinned(1); _heuristics->prepare_for_old_collections(); - // We only excluded pinned regions when we actually add regions to the collection set. + // We only exclude pinned regions when we actually add regions to the collection set. ASSERT_EQ(3UL, _heuristics->unprocessed_old_collection_candidates()); // Here the region is still pinned, so it cannot be added to the collection set. @@ -198,16 +221,18 @@ TEST_VM_F(ShenandoahOldHeuristicTest, skip_pinned_regions) { // The two unpinned regions should be added to the collection set and the pinned // region should be retained at the front of the list of candidates as it would be // likely to become unpinned by the next mixed collection cycle. + EXPECT_TRUE(collection_set_is(0UL, 2UL)); EXPECT_EQ(_collection_set->get_old_garbage(), g1 + g3); EXPECT_EQ(_heuristics->unprocessed_old_collection_candidates(), 1UL); - // Simulate another mixed collection after making region 2 unpinned. This time, + // Simulate another mixed collection after making region 1 unpinned. This time, // the now unpinned region should be added to the collection set. - make_unpinned(2); + make_unpinned(1); _collection_set->clear(); _heuristics->prime_collection_set(_collection_set); EXPECT_EQ(_collection_set->get_old_garbage(), g2); + EXPECT_TRUE(collection_set_is(1UL)); EXPECT_EQ(_heuristics->unprocessed_old_collection_candidates(), 0UL); } @@ -215,22 +240,22 @@ TEST_VM_F(ShenandoahOldHeuristicTest, pinned_region_is_first) { SKIP_IF_NOT_SHENANDOAH(); // Create three old regions with enough garbage to be collected. - size_t g1 = make_garbage_above_threshold(1); - size_t g2 = make_garbage_above_threshold(2); - size_t g3 = make_garbage_above_threshold(3); + size_t g1 = make_garbage_above_threshold(0); + size_t g2 = make_garbage_above_threshold(1); + size_t g3 = make_garbage_above_threshold(2); - make_pinned(1); + make_pinned(0); _heuristics->prepare_for_old_collections(); _heuristics->prime_collection_set(_collection_set); - EXPECT_EQ(_collection_set->get_old_garbage(), g2 + g3); + EXPECT_TRUE(collection_set_is(1UL, 2UL)); EXPECT_EQ(_heuristics->unprocessed_old_collection_candidates(), 1UL); - make_unpinned(1); + make_unpinned(0); _collection_set->clear(); _heuristics->prime_collection_set(_collection_set); - EXPECT_EQ(_collection_set->get_old_garbage(), g1); + EXPECT_TRUE(collection_set_is(0UL)); EXPECT_EQ(_heuristics->unprocessed_old_collection_candidates(), 0UL); } @@ -238,21 +263,23 @@ TEST_VM_F(ShenandoahOldHeuristicTest, pinned_region_is_last) { SKIP_IF_NOT_SHENANDOAH(); // Create three old regions with enough garbage to be collected. - size_t g1 = make_garbage_above_threshold(1); - size_t g2 = make_garbage_above_threshold(2); - size_t g3 = make_garbage_above_threshold(3); + size_t g1 = make_garbage_above_threshold(0); + size_t g2 = make_garbage_above_threshold(1); + size_t g3 = make_garbage_above_threshold(2); - make_pinned(3); + make_pinned(2); _heuristics->prepare_for_old_collections(); _heuristics->prime_collection_set(_collection_set); + EXPECT_TRUE(collection_set_is(0UL, 1UL)); EXPECT_EQ(_collection_set->get_old_garbage(), g1 + g2); EXPECT_EQ(_heuristics->unprocessed_old_collection_candidates(), 1UL); - make_unpinned(3); + make_unpinned(2); _collection_set->clear(); _heuristics->prime_collection_set(_collection_set); + EXPECT_TRUE(collection_set_is(2UL)); EXPECT_EQ(_collection_set->get_old_garbage(), g3); EXPECT_EQ(_heuristics->unprocessed_old_collection_candidates(), 0UL); } @@ -261,25 +288,27 @@ TEST_VM_F(ShenandoahOldHeuristicTest, unpinned_region_is_middle) { SKIP_IF_NOT_SHENANDOAH(); // Create three old regions with enough garbage to be collected. - size_t g1 = make_garbage_above_threshold(1); - size_t g2 = make_garbage_above_threshold(2); - size_t g3 = make_garbage_above_threshold(3); + size_t g1 = make_garbage_above_threshold(0); + size_t g2 = make_garbage_above_threshold(1); + size_t g3 = make_garbage_above_threshold(2); - make_pinned(1); - make_pinned(3); + make_pinned(0); + make_pinned(2); _heuristics->prepare_for_old_collections(); _heuristics->prime_collection_set(_collection_set); + EXPECT_TRUE(collection_set_is(1UL)); EXPECT_EQ(_collection_set->get_old_garbage(), g2); EXPECT_EQ(_heuristics->unprocessed_old_collection_candidates(), 2UL); - make_unpinned(1); - make_unpinned(3); + make_unpinned(0); + make_unpinned(2); _collection_set->clear(); _heuristics->prime_collection_set(_collection_set); + EXPECT_TRUE(collection_set_is(0UL, 2UL)); EXPECT_EQ(_collection_set->get_old_garbage(), g1 + g3); EXPECT_EQ(_heuristics->unprocessed_old_collection_candidates(), 0UL); } -#undef SKIP_IF_NOT_SHENANDOAH \ No newline at end of file +#undef SKIP_IF_NOT_SHENANDOAH From d3679094907d4b5de326a340d06995d06dd5d9e8 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Mon, 20 Mar 2023 17:52:35 +0000 Subject: [PATCH 194/254] Do not visit gc threads during shutdown Reviewed-by: kdnilsen --- src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 802674b1bce50..2d5ca0b752018 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -1801,6 +1801,10 @@ void ShenandoahHeap::prepare_for_verify() { } void ShenandoahHeap::gc_threads_do(ThreadClosure* tcl) const { + if (_shenandoah_policy->is_at_shutdown()) { + return; + } + tcl->do_thread(_control_thread); tcl->do_thread(_regulator_thread); workers()->threads_do(tcl); From 1b6d6b26876f6f456f721c462aab094c57e6ef3a Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Mon, 20 Mar 2023 23:47:40 +0000 Subject: [PATCH 195/254] Fix error in alignment of plabs Reviewed-by: wkemper --- src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index 4bd2f0e64cb27..e610b75f7f1bf 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -287,10 +287,13 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah assert(_heap->mode()->is_generational(), "PLABs are only for generational mode"); // Need to assure that plabs are aligned on multiple of card region. size_t free = r->free(); + // e.g. card_size is 512, card_shift is 9, min_fill_size() is 8 + // free is 514 + // usable_free is 512, which is decreased to 0 size_t usable_free = (free / CardTable::card_size()) << CardTable::card_shift(); if ((free != usable_free) && (free - usable_free < ShenandoahHeap::min_fill_size() * HeapWordSize)) { // We'll have to add another card's memory to the padding - if (usable_free > CardTable::card_size()) { + if (usable_free >= CardTable::card_size()) { usable_free -= CardTable::card_size(); } else { assert(usable_free == 0, "usable_free is a multiple of card_size and card_size > min_fill_size"); From a853a7a4841fbcebce151e3b0e7081bedc782b56 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Tue, 21 Mar 2023 14:44:47 +0000 Subject: [PATCH 196/254] Separate young and old regions in age census at end of each GC Reviewed-by: ysr, wkemper --- .../share/gc/shenandoah/shenandoahEvacTracker.cpp | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.cpp b/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.cpp index 902e7acd7d77d..59771292a562d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.cpp @@ -89,16 +89,23 @@ void ShenandoahEvacuationTracker::print_evacuations_on(outputStream* st, mutators->print_on(st); st->cr(); - AgeTable region_ages(false); + AgeTable young_region_ages(false), old_region_ages(false); ShenandoahHeap* heap = ShenandoahHeap::heap(); for (uint i = 0; i < heap->num_regions(); ++i) { ShenandoahHeapRegion* r = heap->get_region(i); if (r->age() > 0 && r->age() < AgeTable::table_size) { - region_ages.add(r->age(), r->get_live_data_words()); + if (r->is_young()) { + young_region_ages.add(r->age(), r->get_live_data_words()); + } else { + old_region_ages.add(r->age(), r->get_live_data_words()); + } } } - st->print("Regions: "); - region_ages.print_on(st, InitialTenuringThreshold); + st->print("Young regions: "); + young_region_ages.print_on(st, InitialTenuringThreshold); + st->cr(); + st->print("Old regions: "); + old_region_ages.print_on(st, InitialTenuringThreshold); } class ShenandoahStatAggregator : public ThreadClosure { From 1a3dcaa6b2a6a2e386682defe409657eff6591a4 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Thu, 23 Mar 2023 15:51:47 +0000 Subject: [PATCH 197/254] Put regulator thread stop after gc thread iteration is prevented Reviewed-by: ysr, shade, kdnilsen --- src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 2d5ca0b752018..5aeee487a5980 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -2357,21 +2357,21 @@ uint ShenandoahHeap::max_workers() { void ShenandoahHeap::stop() { // The shutdown sequence should be able to terminate when GC is running. - // Step 0a. Stop requesting collections. - regulator_thread()->stop(); - - // Step 0. Notify policy to disable event recording. + // Step 1. Notify policy to disable event recording and prevent visiting gc threads during shutdown _shenandoah_policy->record_shutdown(); - // Step 1. Notify control thread that we are in shutdown. + // Step 2. Stop requesting collections. + regulator_thread()->stop(); + + // Step 3. Notify control thread that we are in shutdown. // Note that we cannot do that with stop(), because stop() is blocking and waits for the actual shutdown. // Doing stop() here would wait for the normal GC cycle to complete, never falling through to cancel below. control_thread()->prepare_for_graceful_shutdown(); - // Step 2. Notify GC workers that we are cancelling GC. + // Step 4. Notify GC workers that we are cancelling GC. cancel_gc(GCCause::_shenandoah_stop_vm); - // Step 3. Wait until GC worker exits normally. + // Step 5. Wait until GC worker exits normally. control_thread()->stop(); } From a5d1a9d8ee70828bf5e2ccdc7a3691eb7ae324ea Mon Sep 17 00:00:00 2001 From: William Kemper Date: Thu, 23 Mar 2023 23:19:37 +0000 Subject: [PATCH 198/254] Lower logging of collection set details to debug and use cset tag Reviewed-by: kdnilsen, ysr --- src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index 98c758a0bf5a6..2be57a2b4ab0d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -747,7 +747,7 @@ void ShenandoahConcurrentGC::op_final_mark() { heap->prepare_concurrent_roots(); if (!heap->collection_set()->is_empty()) { - LogTarget(Info, gc, ergo) lt; + LogTarget(Debug, gc, cset) lt; if (lt.is_enabled()) { ResourceMark rm; LogStream ls(lt); From 2785875de3bf86ab20779c5a24ec7e0f69e452aa Mon Sep 17 00:00:00 2001 From: William Kemper Date: Wed, 29 Mar 2023 22:41:12 +0000 Subject: [PATCH 199/254] Restrict generational mode to adaptive heuristic Reviewed-by: shade --- .../mode/shenandoahGenerationalMode.cpp | 15 +- .../mode/shenandoahGenerationalMode.hpp | 1 + test/hotspot/jtreg/TEST.groups | 16 --- .../jtreg/gc/TestAllocHumongousFragment.java | 17 +++ .../gc/shenandoah/TestAllocIntArrays.java | 9 ++ .../gc/shenandoah/TestAllocObjectArrays.java | 34 +++++ .../jtreg/gc/shenandoah/TestAllocObjects.java | 20 +++ .../gc/shenandoah/TestArrayCopyCheckCast.java | 9 +- .../gc/shenandoah/TestArrayCopyStress.java | 11 +- .../TestDynamicSoftMaxHeapSize.java | 11 ++ .../jtreg/gc/shenandoah/TestElasticTLAB.java | 19 ++- .../jtreg/gc/shenandoah/TestEvilSyncBug.java | 19 ++- .../gc/shenandoah/TestGCThreadGroups.java | 18 +++ .../jtreg/gc/shenandoah/TestHeapUncommit.java | 22 +++ .../gc/shenandoah/TestHumongousThreshold.java | 79 +++++++++++ .../jtreg/gc/shenandoah/TestJcmdHeapDump.java | 12 ++ .../shenandoah/TestLargeObjectAlignment.java | 16 ++- .../jtreg/gc/shenandoah/TestLotsOfCycles.java | 10 ++ .../gc/shenandoah/TestObjItrWithHeapDump.java | 7 +- .../shenandoah/TestParallelRefprocSanity.java | 50 ------- .../jtreg/gc/shenandoah/TestPeriodicGC.java | 45 ++++++ .../TestReferenceRefersToShenandoah.java | 33 ++++- .../TestReferenceShortcutCycle.java | 13 ++ .../gc/shenandoah/TestRefprocSanity.java | 15 ++ .../gc/shenandoah/TestRegionSampling.java | 9 ++ .../shenandoah/TestRegionSamplingLogging.java | 14 +- .../jtreg/gc/shenandoah/TestResizeTLAB.java | 20 +++ .../gc/shenandoah/TestRetainObjects.java | 25 ++++ .../shenandoah/TestShenandoahLogRotation.java | 2 +- .../jtreg/gc/shenandoah/TestSieveObjects.java | 22 +++ .../jtreg/gc/shenandoah/TestSmallHeap.java | 13 +- .../jtreg/gc/shenandoah/TestStringDedup.java | 14 ++ .../gc/shenandoah/TestStringDedupStress.java | 16 +++ .../shenandoah/TestStringInternCleanup.java | 16 +++ .../gc/shenandoah/TestVerifyJCStress.java | 10 ++ .../jtreg/gc/shenandoah/TestVerifyLevels.java | 12 +- .../jtreg/gc/shenandoah/TestWithLogLevel.java | 13 +- .../gc/shenandoah/TestWrongArrayMember.java | 5 +- .../gc/shenandoah/compiler/TestClone.java | 124 +++++++++++++++++ .../shenandoah/compiler/TestReferenceCAS.java | 26 ++++ .../generational/TestCLIModeGenerational.java | 6 +- .../generational/TestConcurrentEvac.java | 6 +- .../generational/TestSimpleGenerational.java | 6 +- .../gc/shenandoah/jni/TestJNICritical.java | 12 +- .../gc/shenandoah/jni/TestJNIGlobalRefs.java | 19 +++ .../gc/shenandoah/jni/TestPinnedGarbage.java | 16 +++ .../gc/shenandoah/jvmti/TestHeapDump.java | 40 ++++++ .../mxbeans/TestChurnNotifications.java | 27 +++- .../shenandoah/mxbeans/TestMemoryMXBeans.java | 15 +- .../shenandoah/mxbeans/TestMemoryPools.java | 11 +- .../mxbeans/TestPauseNotifications.java | 11 ++ .../gc/shenandoah/oom/TestAllocLargeObj.java | 83 ----------- .../oom/TestAllocLargerThanHeap.java | 78 ----------- .../shenandoah/oom/TestAllocOutOfMemory.java | 131 ++++++++++++++++++ .../gc/shenandoah/oom/TestAllocSmallObj.java | 82 ----------- .../shenandoah/oom/TestClassLoaderLeak.java | 7 +- .../gc/shenandoah/oom/TestThreadFailure.java | 14 ++ .../gc/shenandoah/options/TestModeUnlock.java | 1 + .../gcbasher/TestGCBasherWithShenandoah.java | 37 +++++ .../gclocker/TestGCLockerWithShenandoah.java | 18 +++ .../stress/gcold/TestGCOldWithShenandoah.java | 16 +++ ...ssBigAllocationGCEventsWithShenandoah.java | 49 +++++++ .../systemgc/TestSystemGCWithShenandoah.java | 17 +++ 63 files changed, 1194 insertions(+), 350 deletions(-) delete mode 100644 test/hotspot/jtreg/gc/shenandoah/TestParallelRefprocSanity.java delete mode 100644 test/hotspot/jtreg/gc/shenandoah/oom/TestAllocLargeObj.java delete mode 100644 test/hotspot/jtreg/gc/shenandoah/oom/TestAllocLargerThanHeap.java create mode 100644 test/hotspot/jtreg/gc/shenandoah/oom/TestAllocOutOfMemory.java delete mode 100644 test/hotspot/jtreg/gc/shenandoah/oom/TestAllocSmallObj.java create mode 100644 test/hotspot/jtreg/gc/stress/jfr/TestStressBigAllocationGCEventsWithShenandoah.java diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp index 0e83ef59935b0..a34005581385a 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp @@ -23,8 +23,9 @@ */ #include "precompiled.hpp" -#include "gc/shenandoah/mode/shenandoahGenerationalMode.hpp" +#include "gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp" #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" +#include "gc/shenandoah/mode/shenandoahGenerationalMode.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "logging/log.hpp" #include "logging/logTag.hpp" @@ -52,6 +53,18 @@ void ShenandoahGenerationalMode::initialize_flags() const { SHENANDOAH_CHECK_FLAG_SET(ShenandoahCloneBarrier); } +ShenandoahHeuristics* ShenandoahGenerationalMode::initialize_heuristics(ShenandoahGeneration* generation) const { + if (ShenandoahGCHeuristics == nullptr) { + vm_exit_during_initialization("Unknown -XX:ShenandoahGCHeuristics option (null)"); + } + + if (strcmp(ShenandoahGCHeuristics, "adaptive") != 0) { + vm_exit_during_initialization("Generational mode requires the (default) adaptive heuristic"); + } + + return new ShenandoahAdaptiveHeuristics(generation); +} + const char* affiliation_name(oop ptr) { ShenandoahHeap* heap = ShenandoahHeap::heap(); assert(heap->is_in(ptr), "Oop must be in the heap."); diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp index 6527a3b2c41c4..a12567fd3c739 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp @@ -48,6 +48,7 @@ const char* generation_name(GenerationMode mode); class ShenandoahGenerationalMode : public ShenandoahMode { public: virtual void initialize_flags() const; + virtual ShenandoahHeuristics* initialize_heuristics(ShenandoahGeneration* generation) const override; virtual const char* name() { return "Generational"; } virtual bool is_diagnostic() { return false; } virtual bool is_experimental() { return false; } diff --git a/test/hotspot/jtreg/TEST.groups b/test/hotspot/jtreg/TEST.groups index f73694e639bdd..6567d8486a13f 100644 --- a/test/hotspot/jtreg/TEST.groups +++ b/test/hotspot/jtreg/TEST.groups @@ -317,15 +317,6 @@ tier1_gc_gcbasher = \ gc/stress/gcbasher/TestGCBasherWithSerial.java \ gc/stress/gcbasher/TestGCBasherWithParallel.java -tier1_gc_shenandoah_generational = \ - gc/shenandoah/generational/ - -# No tier 2 tests for shenandoah_generational at this time -tier2_gc_shenandoah_generational = - -# No tier 3 tests for shenandoah_generational at this time -tier3_gc_shenandoah_generational = - tier1_gc_shenandoah = \ gc/shenandoah/options/ \ gc/shenandoah/compiler/ \ @@ -357,14 +348,12 @@ tier2_gc_shenandoah = \ -gc/shenandoah/TestStringDedupStress.java \ -:tier1_gc_shenandoah -# include shenandoah generational tests in tier3 shenandoah tier3_gc_shenandoah = \ gc/stress/gcold/TestGCOldWithShenandoah.java \ gc/stress/gcbasher/TestGCBasherWithShenandoah.java \ gc/stress/gclocker/TestGCLockerWithShenandoah.java \ gc/stress/systemgc/TestSystemGCWithShenandoah.java \ gc/shenandoah/TestStringDedupStress.java \ - :hotspot_gc_shenandoah_generational \ -:tier2_gc_shenandoah hotspot_gc_shenandoah = \ @@ -372,11 +361,6 @@ hotspot_gc_shenandoah = \ :tier2_gc_shenandoah \ :tier3_gc_shenandoah -hotspot_gc_shenandoah_generational = \ - :tier1_gc_shenandoah_generational \ - :tier2_gc_shenandoah_generational \ - :tier3_gc_shenandoah_generational - tier1_runtime = \ runtime/ \ -runtime/6626217/bug_21227.java \ diff --git a/test/hotspot/jtreg/gc/TestAllocHumongousFragment.java b/test/hotspot/jtreg/gc/TestAllocHumongousFragment.java index 88b838b4bde43..9a935809c06c1 100644 --- a/test/hotspot/jtreg/gc/TestAllocHumongousFragment.java +++ b/test/hotspot/jtreg/gc/TestAllocHumongousFragment.java @@ -94,6 +94,23 @@ * @run main/othervm -Xmx1g -Xms1g -Xlog:gc -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:ShenandoahTargetNumRegions=2048 * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive * TestAllocHumongousFragment +*/ + +/* + * @test id=generational + * @summary Make sure Shenandoah can recover from humongous allocation fragmentation + * @key randomness + * @requires vm.gc.Shenandoah + * @library /test/lib + * + * @run main/othervm -Xmx1g -Xms1g -Xlog:gc -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:ShenandoahTargetNumRegions=2048 + * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive -XX:ShenandoahGCMode=generational + * -XX:+ShenandoahVerify + * TestAllocHumongousFragment + * + * @run main/othervm -Xmx1g -Xms1g -Xlog:gc -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:ShenandoahTargetNumRegions=2048 + * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive -XX:ShenandoahGCMode=generational + * TestAllocHumongousFragment */ /* diff --git a/test/hotspot/jtreg/gc/shenandoah/TestAllocIntArrays.java b/test/hotspot/jtreg/gc/shenandoah/TestAllocIntArrays.java index 35b0cdfa49ee2..15d8d4683784b 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestAllocIntArrays.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestAllocIntArrays.java @@ -97,6 +97,15 @@ * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -Xmx1g -Xms1g * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive * TestAllocIntArrays + * + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -Xmx1g -Xms1g + * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive -XX:ShenandoahGCMode=generational + * -XX:+ShenandoahVerify + * TestAllocIntArrays + * + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -Xmx1g -Xms1g + * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive -XX:ShenandoahGCMode=generational + * TestAllocIntArrays */ /* diff --git a/test/hotspot/jtreg/gc/shenandoah/TestAllocObjectArrays.java b/test/hotspot/jtreg/gc/shenandoah/TestAllocObjectArrays.java index 8eebba4a30894..f30fda698fa56 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestAllocObjectArrays.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestAllocObjectArrays.java @@ -99,6 +99,35 @@ * TestAllocObjectArrays */ +/* + * @test id=generational + * @summary Acceptance tests: collector can withstand allocation + * @key randomness + * @requires vm.gc.Shenandoah + * @library /test/lib + * + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -Xmx1g -Xms1g + * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive -XX:ShenandoahGCMode=generational + * -XX:+ShenandoahVerify + * TestAllocObjectArrays + * + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -Xmx1g -Xms1g + * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive -XX:ShenandoahGCMode=generational + * TestAllocObjectArrays + * + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -Xmx1g -Xms1g + * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive -XX:ShenandoahGCMode=generational + * -XX:+ShenandoahOOMDuringEvacALot + * -XX:+ShenandoahVerify + * TestAllocObjectArrays + * + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -Xmx1g -Xms1g + * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive -XX:ShenandoahGCMode=generational + * -XX:+ShenandoahAllocFailureALot + * -XX:+ShenandoahVerify + * TestAllocObjectArrays + */ + /* * @test id=static * @summary Acceptance tests: collector can withstand allocation @@ -134,6 +163,11 @@ * -XX:+UseShenandoahGC * -XX:-UseTLAB -XX:+ShenandoahVerify * TestAllocObjectArrays + * + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -Xmx1g -Xms1g + * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive -XX:ShenandoahGCMode=generational + * -XX:-UseTLAB -XX:+ShenandoahVerify + * TestAllocObjectArrays */ /* diff --git a/test/hotspot/jtreg/gc/shenandoah/TestAllocObjects.java b/test/hotspot/jtreg/gc/shenandoah/TestAllocObjects.java index 002991196c7c5..71144e649d757 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestAllocObjects.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestAllocObjects.java @@ -104,6 +104,26 @@ * TestAllocObjects */ +/* + * @test id=generational + * @summary Acceptance tests: collector can withstand allocation + * @requires vm.gc.Shenandoah + * + * @run main/othervm -Xmx1g -Xms1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive -XX:ShenandoahGCMode=generational + * -XX:+ShenandoahVerify + * TestAllocObjects + * + * @run main/othervm -Xmx1g -Xms1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive -XX:ShenandoahGCMode=generational + * TestAllocObjects + * + * @run main/othervm -Xmx1g -Xms1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive -XX:ShenandoahGCMode=generational + * -XX:+ShenandoahSuspendibleWorkers + * TestAllocObjects + */ + /* * @test id=static * @summary Acceptance tests: collector can withstand allocation diff --git a/test/hotspot/jtreg/gc/shenandoah/TestArrayCopyCheckCast.java b/test/hotspot/jtreg/gc/shenandoah/TestArrayCopyCheckCast.java index ecfe5f5836e28..1a89ba41bfa5b 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestArrayCopyCheckCast.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestArrayCopyCheckCast.java @@ -23,11 +23,18 @@ */ /* - * @test + * @test id=default * @requires vm.gc.Shenandoah * * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:TieredStopAtLevel=0 -Xmx16m TestArrayCopyCheckCast */ + +/* + * @test id=generational + * @requires vm.gc.Shenandoah + * + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:TieredStopAtLevel=0 -Xmx16m TestArrayCopyCheckCast -XX:ShenandoahGCMode=generational + */ public class TestArrayCopyCheckCast { static class Foo {} diff --git a/test/hotspot/jtreg/gc/shenandoah/TestArrayCopyStress.java b/test/hotspot/jtreg/gc/shenandoah/TestArrayCopyStress.java index 6a00b0f51de08..7ebcd964fe83d 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestArrayCopyStress.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestArrayCopyStress.java @@ -26,13 +26,22 @@ import jdk.test.lib.Utils; /* - * @test + * @test id=default * @key randomness * @requires vm.gc.Shenandoah * @library /test/lib * * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:TieredStopAtLevel=0 -Xmx16m TestArrayCopyStress */ + +/* + * @test id=generational + * @key randomness + * @requires vm.gc.Shenandoah + * @library /test/lib + * + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -XX:TieredStopAtLevel=0 -Xmx16m TestArrayCopyStress + */ public class TestArrayCopyStress { private static final int ARRAY_SIZE = 1000; diff --git a/test/hotspot/jtreg/gc/shenandoah/TestDynamicSoftMaxHeapSize.java b/test/hotspot/jtreg/gc/shenandoah/TestDynamicSoftMaxHeapSize.java index f34fcd8024da5..97a425d2e22dc 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestDynamicSoftMaxHeapSize.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestDynamicSoftMaxHeapSize.java @@ -62,6 +62,17 @@ * TestDynamicSoftMaxHeapSize */ +/* + * @test id=generational + * @requires vm.gc.Shenandoah + * @library /test/lib + * + * @run main/othervm -Xms16m -Xmx512m -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive -XX:ShenandoahGCMode=generational + * -Dtarget=10000 + * TestDynamicSoftMaxHeapSize + */ + /* * @test id=static * @requires vm.gc.Shenandoah diff --git a/test/hotspot/jtreg/gc/shenandoah/TestElasticTLAB.java b/test/hotspot/jtreg/gc/shenandoah/TestElasticTLAB.java index b498228a9329d..f46a419e3a6e2 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestElasticTLAB.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestElasticTLAB.java @@ -23,7 +23,7 @@ */ /* - * @test + * @test id=default * @key randomness * @summary Test that Shenandoah is able to work with elastic TLABs * @requires vm.gc.Shenandoah @@ -39,6 +39,23 @@ * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -XX:+UseTLAB -XX:+ShenandoahElasticTLAB TestElasticTLAB */ +/* + * @test id=generational + * @key randomness + * @summary Test that Shenandoah is able to work with elastic TLABs + * @requires vm.gc.Shenandoah + * @library /test/lib + * + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xmx1g -XX:-UseTLAB -XX:-ShenandoahElasticTLAB -XX:+ShenandoahVerify TestElasticTLAB + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xmx1g -XX:-UseTLAB -XX:-ShenandoahElasticTLAB TestElasticTLAB + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xmx1g -XX:-UseTLAB -XX:+ShenandoahElasticTLAB -XX:+ShenandoahVerify TestElasticTLAB + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xmx1g -XX:-UseTLAB -XX:+ShenandoahElasticTLAB TestElasticTLAB + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xmx1g -XX:+UseTLAB -XX:-ShenandoahElasticTLAB -XX:+ShenandoahVerify TestElasticTLAB + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xmx1g -XX:+UseTLAB -XX:-ShenandoahElasticTLAB TestElasticTLAB + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xmx1g -XX:+UseTLAB -XX:+ShenandoahElasticTLAB -XX:+ShenandoahVerify TestElasticTLAB + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xmx1g -XX:+UseTLAB -XX:+ShenandoahElasticTLAB TestElasticTLAB + */ + import java.util.Random; import jdk.test.lib.Utils; diff --git a/test/hotspot/jtreg/gc/shenandoah/TestEvilSyncBug.java b/test/hotspot/jtreg/gc/shenandoah/TestEvilSyncBug.java index e58e8d6db5ce0..f4132c114f303 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestEvilSyncBug.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestEvilSyncBug.java @@ -23,13 +23,23 @@ */ /* - * @test + * @test id=default * @summary Tests for crash/assert when attaching init thread during shutdown * @requires vm.gc.Shenandoah * @library /test/lib * @modules java.base/jdk.internal.misc * java.management - * @run driver/timeout=480 TestEvilSyncBug + * @run driver/timeout=480 TestEvilSyncBug -XX:ShenandoahGCHeuristics=aggressive + */ + +/* + * @test id=generational + * @summary Tests for crash/assert when attaching init thread during shutdown + * @requires vm.gc.Shenandoah + * @library /test/lib + * @modules java.base/jdk.internal.misc + * java.management + * @run driver/timeout=480 TestEvilSyncBug -XX:ShenandoahGCMode=generational */ import java.util.*; @@ -46,9 +56,10 @@ public class TestEvilSyncBug { static Thread[] hooks = new MyHook[10000]; public static void main(String[] args) throws Exception { - if (args.length > 0) { + if ("test".equals(args[0])) { test(); } else { + String options = args[0]; // Use 1/4 of available processors to avoid over-saturation. int numJobs = Math.max(1, Runtime.getRuntime().availableProcessors() / 4); ExecutorService pool = Executors.newFixedThreadPool(numJobs); @@ -61,7 +72,7 @@ public static void main(String[] args) throws Exception { "-XX:+UnlockExperimentalVMOptions", "-XX:+UnlockDiagnosticVMOptions", "-XX:+UseShenandoahGC", - "-XX:ShenandoahGCHeuristics=aggressive", + options, "TestEvilSyncBug", "test"); OutputAnalyzer output = new OutputAnalyzer(pb.start()); output.shouldHaveExitValue(0); diff --git a/test/hotspot/jtreg/gc/shenandoah/TestGCThreadGroups.java b/test/hotspot/jtreg/gc/shenandoah/TestGCThreadGroups.java index ff1596d6833d2..4a3cd80f13781 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestGCThreadGroups.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestGCThreadGroups.java @@ -76,6 +76,24 @@ * TestGCThreadGroups */ +/** + * @test id=generational + * @summary Test Shenandoah GC uses concurrent/parallel threads correctly + * @requires vm.gc.Shenandoah + * + * @run main/othervm -Xmx16m -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * -XX:+UseShenandoahGC + * -XX:ConcGCThreads=2 -XX:ParallelGCThreads=4 + * -Dtarget=1000 -XX:ShenandoahGCMode=generational + * TestGCThreadGroups + * + * @run main/othervm -Xmx16m -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * -XX:+UseShenandoahGC + * -XX:-UseDynamicNumberOfGCThreads + * -Dtarget=1000 -XX:ShenandoahGCMode=generational + * TestGCThreadGroups + */ + /** * @test id=iu * @summary Test Shenandoah GC uses concurrent/parallel threads correctly diff --git a/test/hotspot/jtreg/gc/shenandoah/TestHeapUncommit.java b/test/hotspot/jtreg/gc/shenandoah/TestHeapUncommit.java index 6c54a34dae551..bd97e1f861ccb 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestHeapUncommit.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestHeapUncommit.java @@ -84,6 +84,28 @@ * TestHeapUncommit */ +/* + * @test id=generational + * @summary Acceptance tests: collector can withstand allocation + * @key randomness + * @requires vm.gc.Shenandoah + * @library /test/lib + * + * @run main/othervm -Xmx1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+ShenandoahUncommit -XX:ShenandoahUncommitDelay=0 + * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive -XX:ShenandoahGCMode=generational + * -XX:+ShenandoahVerify + * TestHeapUncommit + * + * @run main/othervm -Xmx1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+ShenandoahUncommit -XX:ShenandoahUncommitDelay=0 + * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive -XX:ShenandoahGCMode=generational + * TestHeapUncommit + * + * @run main/othervm -Xmx1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+ShenandoahUncommit -XX:ShenandoahUncommitDelay=0 + * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive -XX:ShenandoahGCMode=generational + * -XX:-UseTLAB -XX:+ShenandoahVerify + * TestHeapUncommit + */ + /* * @test id=iu * @summary Acceptance tests: collector can withstand allocation diff --git a/test/hotspot/jtreg/gc/shenandoah/TestHumongousThreshold.java b/test/hotspot/jtreg/gc/shenandoah/TestHumongousThreshold.java index 845a6617ebd63..8f76cbff629ef 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestHumongousThreshold.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestHumongousThreshold.java @@ -108,6 +108,85 @@ * TestHumongousThreshold */ +/* + * @test id=generational + * @key randomness + * @requires vm.gc.Shenandoah + * @library /test/lib + * + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xmx1g + * -XX:+ShenandoahVerify + * TestHumongousThreshold + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xmx1g + * -XX:+ShenandoahVerify -XX:ShenandoahHumongousThreshold=50 + * TestHumongousThreshold + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xmx1g + * -XX:+ShenandoahVerify -XX:ShenandoahHumongousThreshold=90 + * TestHumongousThreshold + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xmx1g + * -XX:+ShenandoahVerify -XX:ShenandoahHumongousThreshold=99 + * TestHumongousThreshold + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xmx1g + * -XX:+ShenandoahVerify -XX:ShenandoahHumongousThreshold=100 + * TestHumongousThreshold + * + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xmx1g + * -XX:-UseTLAB -XX:+ShenandoahVerify + * TestHumongousThreshold + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xmx1g + * -XX:-UseTLAB -XX:+ShenandoahVerify -XX:ShenandoahHumongousThreshold=50 + * TestHumongousThreshold + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xmx1g + * -XX:-UseTLAB -XX:+ShenandoahVerify -XX:ShenandoahHumongousThreshold=90 + * TestHumongousThreshold + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xmx1g + * -XX:-UseTLAB -XX:+ShenandoahVerify -XX:ShenandoahHumongousThreshold=99 + * TestHumongousThreshold + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xmx1g + * -XX:-UseTLAB -XX:+ShenandoahVerify -XX:ShenandoahHumongousThreshold=100 + * TestHumongousThreshold + */ + +/* + * @test id=generational-16b + * @key randomness + * @requires vm.gc.Shenandoah + * @requires vm.bits == "64" + * @library /test/lib + * + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xmx1g + * -XX:ObjectAlignmentInBytes=16 -XX:+ShenandoahVerify + * TestHumongousThreshold + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xmx1g + * -XX:ObjectAlignmentInBytes=16 -XX:+ShenandoahVerify -XX:ShenandoahHumongousThreshold=50 + * TestHumongousThreshold + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xmx1g + * -XX:ObjectAlignmentInBytes=16 -XX:+ShenandoahVerify -XX:ShenandoahHumongousThreshold=90 + * TestHumongousThreshold + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xmx1g + * -XX:ObjectAlignmentInBytes=16 -XX:+ShenandoahVerify -XX:ShenandoahHumongousThreshold=99 + * TestHumongousThreshold + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xmx1g + * -XX:ObjectAlignmentInBytes=16 -XX:+ShenandoahVerify -XX:ShenandoahHumongousThreshold=100 + * TestHumongousThreshold + * + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xmx1g + * -XX:-UseTLAB -XX:ObjectAlignmentInBytes=16 -XX:+ShenandoahVerify + * TestHumongousThreshold + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xmx1g + * -XX:-UseTLAB -XX:ObjectAlignmentInBytes=16 -XX:+ShenandoahVerify -XX:ShenandoahHumongousThreshold=50 + * TestHumongousThreshold + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xmx1g + * -XX:-UseTLAB -XX:ObjectAlignmentInBytes=16 -XX:+ShenandoahVerify -XX:ShenandoahHumongousThreshold=90 + * TestHumongousThreshold + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xmx1g + * -XX:-UseTLAB -XX:ObjectAlignmentInBytes=16 -XX:+ShenandoahVerify -XX:ShenandoahHumongousThreshold=99 + * TestHumongousThreshold + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xmx1g + * -XX:-UseTLAB -XX:ObjectAlignmentInBytes=16 -XX:+ShenandoahVerify -XX:ShenandoahHumongousThreshold=100 + * TestHumongousThreshold + */ + import java.util.Random; import jdk.test.lib.Utils; diff --git a/test/hotspot/jtreg/gc/shenandoah/TestJcmdHeapDump.java b/test/hotspot/jtreg/gc/shenandoah/TestJcmdHeapDump.java index cc5bc5d425e83..0ec81ec6e1884 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestJcmdHeapDump.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestJcmdHeapDump.java @@ -72,6 +72,18 @@ * TestJcmdHeapDump */ +/* + * @test id=generational + * @library /test/lib + * @modules jdk.attach/com.sun.tools.attach + * @requires vm.gc.Shenandoah + * + * @run main/othervm/timeout=480 -Xmx16m -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive -XX:ShenandoahGCMode=generational + * -Dtarget=10000 + * TestJcmdHeapDump + */ + /* * @test id=static * @library /test/lib diff --git a/test/hotspot/jtreg/gc/shenandoah/TestLargeObjectAlignment.java b/test/hotspot/jtreg/gc/shenandoah/TestLargeObjectAlignment.java index 893014804e295..2ace318a299f4 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestLargeObjectAlignment.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestLargeObjectAlignment.java @@ -23,7 +23,7 @@ */ /* - * @test + * @test default * @summary Shenandoah crashes with -XX:ObjectAlignmentInBytes=16 * @key randomness * @requires vm.gc.Shenandoah @@ -36,6 +36,20 @@ * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ObjectAlignmentInBytes=16 -XX:TieredStopAtLevel=4 TestLargeObjectAlignment */ +/* + * @test generational + * @summary Shenandoah crashes with -XX:ObjectAlignmentInBytes=16 + * @key randomness + * @requires vm.gc.Shenandoah + * @requires vm.bits == "64" + * @library /test/lib + * + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -XX:ObjectAlignmentInBytes=16 -Xint TestLargeObjectAlignment + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -XX:ObjectAlignmentInBytes=16 -XX:-TieredCompilation TestLargeObjectAlignment + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -XX:ObjectAlignmentInBytes=16 -XX:TieredStopAtLevel=1 TestLargeObjectAlignment + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -XX:ObjectAlignmentInBytes=16 -XX:TieredStopAtLevel=4 TestLargeObjectAlignment + */ + import java.util.ArrayList; import java.util.List; import java.util.Random; diff --git a/test/hotspot/jtreg/gc/shenandoah/TestLotsOfCycles.java b/test/hotspot/jtreg/gc/shenandoah/TestLotsOfCycles.java index 63d9fa08767d3..ca32a1023ec4e 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestLotsOfCycles.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestLotsOfCycles.java @@ -71,6 +71,16 @@ * TestLotsOfCycles */ +/* + * @test id=generational + * @requires vm.gc.Shenandoah + * + * @run main/othervm/timeout=480 -Xmx16m -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive -XX:ShenandoahGCMode=generational + * -Dtarget=10000 + * TestLotsOfCycles + */ + /* * @test id=static * @requires vm.gc.Shenandoah diff --git a/test/hotspot/jtreg/gc/shenandoah/TestObjItrWithHeapDump.java b/test/hotspot/jtreg/gc/shenandoah/TestObjItrWithHeapDump.java index 41943d4befd71..dea81f9754ca0 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestObjItrWithHeapDump.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestObjItrWithHeapDump.java @@ -56,9 +56,10 @@ public static void main(String[] args) throws Exception { } String[][][] modeHeuristics = new String[][][] { - {{"satb"}, {"adaptive", "compact", "static", "aggressive"}}, - {{"iu"}, {"adaptive", "aggressive"}}, - {{"passive"}, {"passive"}} + {{"satb"}, {"adaptive", "compact", "static", "aggressive"}}, + {{"generational"}, {"adaptive"}}, + {{"iu"}, {"adaptive", "aggressive"}}, + {{"passive"}, {"passive"}} }; for (String[][] mh : modeHeuristics) { diff --git a/test/hotspot/jtreg/gc/shenandoah/TestParallelRefprocSanity.java b/test/hotspot/jtreg/gc/shenandoah/TestParallelRefprocSanity.java deleted file mode 100644 index 474831a477161..0000000000000 --- a/test/hotspot/jtreg/gc/shenandoah/TestParallelRefprocSanity.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2017, 2018, Red Hat, Inc. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -/* - * @test - * @summary Test that reference processing works with both parallel and non-parallel variants. - * @requires vm.gc.Shenandoah - * - * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g TestParallelRefprocSanity - * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:-ParallelRefProcEnabled TestParallelRefprocSanity - * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g -XX:+ParallelRefProcEnabled TestParallelRefprocSanity - */ - -import java.lang.ref.*; - -public class TestParallelRefprocSanity { - - static final long TARGET_MB = Long.getLong("target", 10_000); // 10 Gb allocation - - static volatile Object sink; - - public static void main(String[] args) throws Exception { - long count = TARGET_MB * 1024 * 1024 / 32; - for (long c = 0; c < count; c++) { - sink = new WeakReference(new Object()); - } - } - -} diff --git a/test/hotspot/jtreg/gc/shenandoah/TestPeriodicGC.java b/test/hotspot/jtreg/gc/shenandoah/TestPeriodicGC.java index db7930ea0477b..1eeb904a19a63 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestPeriodicGC.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestPeriodicGC.java @@ -54,6 +54,31 @@ public static void testWith(String msg, boolean periodic, String... args) throws } } + public static void testGenerational(boolean periodic, String... args) throws Exception { + String[] cmds = Arrays.copyOf(args, args.length + 2); + cmds[args.length] = TestPeriodicGC.class.getName(); + cmds[args.length + 1] = "test"; + ProcessBuilder pb = ProcessTools.createJavaProcessBuilder(cmds); + + OutputAnalyzer output = new OutputAnalyzer(pb.start()); + output.shouldHaveExitValue(0); + if (periodic) { + if (!output.getOutput().contains("Trigger (YOUNG): Time since last GC")) { + throw new AssertionError("Generational mode: Should have periodic young GC in logs"); + } + if (!output.getOutput().contains("Trigger (OLD): Time since last GC")) { + throw new AssertionError("Generational mode: Should have periodic old GC in logs"); + } + } else { + if (output.getOutput().contains("Trigger (YOUNG): Time since last GC")) { + throw new AssertionError("Generational mode: Should not have periodic young GC in logs"); + } + if (output.getOutput().contains("Trigger (OLD): Time since last GC")) { + throw new AssertionError("Generational mode: Should not have periodic old GC in logs"); + } + } + } + public static void main(String[] args) throws Exception { if (args.length > 0 && args[0].equals("test")) { Thread.sleep(5000); // stay idle @@ -157,6 +182,26 @@ public static void main(String[] args) throws Exception { "-XX:ShenandoahGCMode=passive", "-XX:ShenandoahGuaranteedGCInterval=1000" ); + + testGenerational(true, + "-Xlog:gc", + "-XX:+UnlockDiagnosticVMOptions", + "-XX:+UnlockExperimentalVMOptions", + "-XX:+UseShenandoahGC", + "-XX:ShenandoahGCMode=generational", + "-XX:ShenandoahGuaranteedYoungGCInterval=1000", + "-XX:ShenandoahGuaranteedOldGCInterval=1500" + ); + + testGenerational(false, + "-Xlog:gc", + "-XX:+UnlockDiagnosticVMOptions", + "-XX:+UnlockExperimentalVMOptions", + "-XX:+UseShenandoahGC", + "-XX:ShenandoahGCMode=generational", + "-XX:ShenandoahGuaranteedYoungGCInterval=0", + "-XX:ShenandoahGuaranteedOldGCInterval=0" + ); } } diff --git a/test/hotspot/jtreg/gc/shenandoah/TestReferenceRefersToShenandoah.java b/test/hotspot/jtreg/gc/shenandoah/TestReferenceRefersToShenandoah.java index 8bad2fdbfa4e4..6fd758e72b0f9 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestReferenceRefersToShenandoah.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestReferenceRefersToShenandoah.java @@ -35,7 +35,20 @@ * gc.shenandoah.TestReferenceRefersToShenandoah */ -/* @test id=iu +/* @test id=satb-100 + * @requires vm.gc.Shenandoah + * @library /test/lib + * @build jdk.test.whitebox.WhiteBox + * @modules java.base + * @run main jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox + * @run main/othervm + * -Xbootclasspath/a:. + * -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI + * -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=satb -XX:ShenandoahGarbageThreshold=100 -Xmx100m + * gc.shenandoah.TestReferenceRefersToShenandoah + */ + +/* @test id=generational * @requires vm.gc.Shenandoah * @library /test/lib * @build jdk.test.whitebox.WhiteBox @@ -43,11 +56,11 @@ * @run main/othervm * -Xbootclasspath/a:. * -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI - * -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=iu + * -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational * gc.shenandoah.TestReferenceRefersToShenandoah */ -/* @test id=satb-100 +/* @test id=generational-100 * @requires vm.gc.Shenandoah * @library /test/lib * @build jdk.test.whitebox.WhiteBox @@ -56,7 +69,19 @@ * @run main/othervm * -Xbootclasspath/a:. * -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI - * -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=satb -XX:ShenandoahGarbageThreshold=100 -Xmx100m + * -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -XX:ShenandoahGarbageThreshold=100 -Xmx100m + * gc.shenandoah.TestReferenceRefersToShenandoah + */ + +/* @test id=iu + * @requires vm.gc.Shenandoah + * @library /test/lib + * @build jdk.test.whitebox.WhiteBox + * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox + * @run main/othervm + * -Xbootclasspath/a:. + * -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI + * -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=iu * gc.shenandoah.TestReferenceRefersToShenandoah */ diff --git a/test/hotspot/jtreg/gc/shenandoah/TestReferenceShortcutCycle.java b/test/hotspot/jtreg/gc/shenandoah/TestReferenceShortcutCycle.java index 1c2c2ed5ca7a9..7522723d2afe3 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestReferenceShortcutCycle.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestReferenceShortcutCycle.java @@ -36,6 +36,19 @@ * gc.shenandoah.TestReferenceShortcutCycle */ +/* @test id=generational-100 + * @requires vm.gc.Shenandoah + * @library /test/lib + * @build jdk.test.whitebox.WhiteBox + * @modules java.base + * @run main jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox + * @run main/othervm + * -Xbootclasspath/a:. + * -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI + * -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -XX:ShenandoahGarbageThreshold=100 -Xmx100m + * gc.shenandoah.TestReferenceShortcutCycle + */ + /* @test id=iu-100 * @requires vm.gc.Shenandoah * @library /test/lib diff --git a/test/hotspot/jtreg/gc/shenandoah/TestRefprocSanity.java b/test/hotspot/jtreg/gc/shenandoah/TestRefprocSanity.java index 03f008d10c395..ffb195c76f5b0 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestRefprocSanity.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestRefprocSanity.java @@ -41,6 +41,21 @@ * TestRefprocSanity */ +/* + * @test id=generational + * @summary Test that null references/referents work fine + * @requires vm.gc.Shenandoah + * + * @run main/othervm -Xmx128m -Xms128m -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * -XX:+ShenandoahVerify + * TestRefprocSanity + * + * @run main/othervm -Xmx128m -Xms128m -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * TestRefprocSanity + */ + /* * @test id=iu * @summary Test that null references/referents work fine diff --git a/test/hotspot/jtreg/gc/shenandoah/TestRegionSampling.java b/test/hotspot/jtreg/gc/shenandoah/TestRegionSampling.java index dd98585181ce2..2ceb2e590adfa 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestRegionSampling.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestRegionSampling.java @@ -46,6 +46,15 @@ * TestRegionSampling */ +/* + * @test id=generational + * @requires vm.gc.Shenandoah + * + * @run main/othervm -Xmx1g -Xms1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+ShenandoahRegionSampling + * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive -XX:ShenandoahGCMode=generational + * TestRegionSampling + */ + /* * @test id=static * @requires vm.gc.Shenandoah diff --git a/test/hotspot/jtreg/gc/shenandoah/TestRegionSamplingLogging.java b/test/hotspot/jtreg/gc/shenandoah/TestRegionSamplingLogging.java index c985bbc310dd0..6c891d9a2dcbb 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestRegionSamplingLogging.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestRegionSamplingLogging.java @@ -23,16 +23,26 @@ */ /* - * @test id=rotation + * @test id=default-rotation * @requires vm.gc.Shenandoah * * @run main/othervm -Xmx1g -Xms1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions * -XX:+ShenandoahRegionSampling -XX:+ShenandoahRegionSampling - * -Xlog:gc+region=debug:region-snapshots-%p.log::filesize=100,filecount=3 + * -Xlog:gc+region=trace:region-snapshots-%p.log::filesize=100,filecount=3 * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive * TestRegionSamplingLogging */ +/* + * @test id=generational-rotation + * @requires vm.gc.Shenandoah + * + * @run main/othervm -Xmx1g -Xms1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * -XX:+ShenandoahRegionSampling -XX:+ShenandoahRegionSampling + * -Xlog:gc+region=trace:region-snapshots-%p.log::filesize=100,filecount=3 + * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive -XX:ShenandoahGCMode=generational + * TestRegionSamplingLogging + */ import java.io.File; import java.util.Arrays; diff --git a/test/hotspot/jtreg/gc/shenandoah/TestResizeTLAB.java b/test/hotspot/jtreg/gc/shenandoah/TestResizeTLAB.java index 095f99395693b..896a869c5c932 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestResizeTLAB.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestResizeTLAB.java @@ -98,6 +98,26 @@ * TestResizeTLAB */ +/* + * @test id=generational + * @key randomness + * @summary Test that Shenandoah is able to work with(out) resizeable TLABs + * @requires vm.gc.Shenandoah + * @library /test/lib + * + * @run main/othervm -Xmx1g -Xms1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive -XX:ShenandoahGCMode=generational + * -XX:+ShenandoahVerify + * -XX:+ResizeTLAB + * TestResizeTLAB + * + * @run main/othervm -Xmx1g -Xms1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive -XX:ShenandoahGCMode=generational + * -XX:+ShenandoahVerify + * -XX:-ResizeTLAB + * TestResizeTLAB + */ + /* * @test id=static * @key randomness diff --git a/test/hotspot/jtreg/gc/shenandoah/TestRetainObjects.java b/test/hotspot/jtreg/gc/shenandoah/TestRetainObjects.java index 7a28548f610c3..90a28670b3797 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestRetainObjects.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestRetainObjects.java @@ -83,6 +83,31 @@ * TestRetainObjects */ +/* + * @test id=generational + * @summary Acceptance tests: collector can deal with retained objects + * @requires vm.gc.Shenandoah + * + * @run main/othervm -Xmx1g -Xms1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive -XX:ShenandoahGCMode=generational + * -XX:+ShenandoahVerify + * TestRetainObjects + * + * @run main/othervm -Xmx1g -Xms1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive -XX:ShenandoahGCMode=generational + * TestRetainObjects + * + * @run main/othervm -Xmx1g -Xms1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive -XX:ShenandoahGCMode=generational + * -XX:+ShenandoahOOMDuringEvacALot -XX:+ShenandoahVerify + * TestRetainObjects + * + * @run main/othervm -Xmx1g -Xms1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive -XX:ShenandoahGCMode=generational + * -XX:+ShenandoahAllocFailureALot -XX:+ShenandoahVerify + * TestRetainObjects + */ + /* * @test id=static * @summary Acceptance tests: collector can deal with retained objects diff --git a/test/hotspot/jtreg/gc/shenandoah/TestShenandoahLogRotation.java b/test/hotspot/jtreg/gc/shenandoah/TestShenandoahLogRotation.java index 54ce716d785e8..9b9d633ef88da 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestShenandoahLogRotation.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestShenandoahLogRotation.java @@ -28,7 +28,7 @@ * * @run main/othervm -Xmx1g -Xms1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions * -XX:+ShenandoahRegionSampling -XX:+ShenandoahRegionSampling - * -Xlog:gc+region=debug:region-snapshots-%p.log::filesize=100,filecount=3 + * -Xlog:gc+region=trace:region-snapshots-%p.log::filesize=100,filecount=3 * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive * TestShenandoahLogRotation */ diff --git a/test/hotspot/jtreg/gc/shenandoah/TestSieveObjects.java b/test/hotspot/jtreg/gc/shenandoah/TestSieveObjects.java index 1b16ba4b8d2e7..bb63fb886628b 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestSieveObjects.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestSieveObjects.java @@ -90,6 +90,28 @@ * */ +/* + * @test id=generational + * @summary Acceptance tests: collector can deal with retained objects + * @key randomness + * @requires vm.gc.Shenandoah + * @library /test/lib + * + * @run main/othervm -Xmx1g -Xms1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive -XX:ShenandoahGCMode=generational + * -XX:+ShenandoahOOMDuringEvacALot -XX:+ShenandoahVerify + * TestSieveObjects + * + * @run main/othervm -Xmx1g -Xms1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive -XX:ShenandoahGCMode=generational + * -XX:+ShenandoahAllocFailureALot -XX:+ShenandoahVerify + * TestSieveObjects + * + * @run main/othervm -Xmx1g -Xms1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive -XX:ShenandoahGCMode=generational + * TestSieveObjects + */ + /* * @test id=static * @summary Acceptance tests: collector can deal with retained objects diff --git a/test/hotspot/jtreg/gc/shenandoah/TestSmallHeap.java b/test/hotspot/jtreg/gc/shenandoah/TestSmallHeap.java index 56f416790f5bc..c629d038df5a0 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestSmallHeap.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestSmallHeap.java @@ -23,7 +23,7 @@ */ /* - * @test + * @test default * @requires vm.gc.Shenandoah * * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC TestSmallHeap @@ -34,6 +34,17 @@ * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx4m TestSmallHeap */ +/* + * @test generational + * @requires vm.gc.Shenandoah + * + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational TestSmallHeap + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xmx64m TestSmallHeap + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xmx32m TestSmallHeap + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xmx16m TestSmallHeap + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xmx8m TestSmallHeap + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xmx4m TestSmallHeap + */ public class TestSmallHeap { public static void main(String[] args) throws Exception { diff --git a/test/hotspot/jtreg/gc/shenandoah/TestStringDedup.java b/test/hotspot/jtreg/gc/shenandoah/TestStringDedup.java index 94691432f38fb..4356a80fb8757 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestStringDedup.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestStringDedup.java @@ -65,6 +65,20 @@ * TestStringDedup */ +/* + * @test id=generational + * @summary Test Shenandoah string deduplication implementation + * @key randomness + * @requires vm.gc.Shenandoah + * @library /test/lib + * @modules java.base/java.lang:open + * java.management + * + * @run main/othervm -Xmx256m -Xlog:gc+stats -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseStringDeduplication + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -XX:StringDeduplicationAgeThreshold=3 + * TestStringDedup + */ + /* * @test id=iu * @summary Test Shenandoah string deduplication implementation diff --git a/test/hotspot/jtreg/gc/shenandoah/TestStringDedupStress.java b/test/hotspot/jtreg/gc/shenandoah/TestStringDedupStress.java index 90b383d05871f..c6a6ca31785d6 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestStringDedupStress.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestStringDedupStress.java @@ -42,6 +42,22 @@ * TestStringDedupStress */ +/* + * @test id=generational + * @summary Test Shenandoah string deduplication implementation + * @key randomness + * @requires vm.gc.Shenandoah + * @library /test/lib + * @modules java.base/java.lang:open + * java.management + * + * @run main/othervm -Xmx1g -Xlog:gc+stats -Xlog:gc -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseStringDeduplication + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * -XX:+ShenandoahDegeneratedGC + * -DtargetStrings=3000000 + * TestStringDedupStress + */ + /* * @test id=default * @summary Test Shenandoah string deduplication implementation diff --git a/test/hotspot/jtreg/gc/shenandoah/TestStringInternCleanup.java b/test/hotspot/jtreg/gc/shenandoah/TestStringInternCleanup.java index cf663f2329cb8..98e6e75d064ba 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestStringInternCleanup.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestStringInternCleanup.java @@ -75,6 +75,22 @@ * TestStringInternCleanup */ +/* + * @test id=generational + * @summary Check that Shenandoah cleans up interned strings + * @requires vm.gc.Shenandoah + * + * @run main/othervm -Xmx64m -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+ClassUnloadingWithConcurrentMark + * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive -XX:ShenandoahGCMode=generational + * -XX:+ShenandoahVerify + * TestStringInternCleanup + * + * @run main/othervm -Xmx64m -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+ClassUnloadingWithConcurrentMark + * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive -XX:ShenandoahGCMode=generational + * TestStringInternCleanup + */ + + /* * @test id=iu * @summary Check that Shenandoah cleans up interned strings diff --git a/test/hotspot/jtreg/gc/shenandoah/TestVerifyJCStress.java b/test/hotspot/jtreg/gc/shenandoah/TestVerifyJCStress.java index 435c4c25004ba..bfc5c24ebea56 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestVerifyJCStress.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestVerifyJCStress.java @@ -57,6 +57,11 @@ * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=compact * -XX:+ShenandoahVerify -XX:+ShenandoahVerifyOptoBarriers * TestVerifyJCStress + * + * @run main/othervm -Xmx1g -Xms1g -XX:+UnlockExperimentalVMOptions -XX:+UnlockDiagnosticVMOptions + * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive -XX:ShenandoahGCMode=generational + * -XX:+ShenandoahVerify -XX:+ShenandoahVerifyOptoBarriers + * TestVerifyJCStress */ /* @@ -76,6 +81,11 @@ * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=compact * -XX:+ShenandoahVerify * TestVerifyJCStress + * + * @run main/othervm -Xmx1g -Xms1g -XX:+UnlockExperimentalVMOptions -XX:+UnlockDiagnosticVMOptions + * -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=adaptive -XX:ShenandoahGCMode=generational + * -XX:+ShenandoahVerify + * TestVerifyJCStress */ /* diff --git a/test/hotspot/jtreg/gc/shenandoah/TestVerifyLevels.java b/test/hotspot/jtreg/gc/shenandoah/TestVerifyLevels.java index 880a965010feb..df5d9c92586c3 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestVerifyLevels.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestVerifyLevels.java @@ -23,7 +23,7 @@ */ /* - * @test + * @test default * @requires vm.gc.Shenandoah * * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:+UnlockDiagnosticVMOptions -Xmx128m -XX:+ShenandoahVerify -XX:ShenandoahVerifyLevel=0 TestVerifyLevels @@ -33,6 +33,16 @@ * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:+UnlockDiagnosticVMOptions -Xmx128m -XX:+ShenandoahVerify -XX:ShenandoahVerifyLevel=4 TestVerifyLevels */ +/* + * @test generational + * @requires vm.gc.Shenandoah + * + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -XX:+UnlockDiagnosticVMOptions -Xmx128m -XX:+ShenandoahVerify -XX:ShenandoahVerifyLevel=0 TestVerifyLevels + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -XX:+UnlockDiagnosticVMOptions -Xmx128m -XX:+ShenandoahVerify -XX:ShenandoahVerifyLevel=1 TestVerifyLevels + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -XX:+UnlockDiagnosticVMOptions -Xmx128m -XX:+ShenandoahVerify -XX:ShenandoahVerifyLevel=2 TestVerifyLevels + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -XX:+UnlockDiagnosticVMOptions -Xmx128m -XX:+ShenandoahVerify -XX:ShenandoahVerifyLevel=3 TestVerifyLevels + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -XX:+UnlockDiagnosticVMOptions -Xmx128m -XX:+ShenandoahVerify -XX:ShenandoahVerifyLevel=4 TestVerifyLevels + */ public class TestVerifyLevels { static final long TARGET_MB = Long.getLong("target", 1_000); // 1 Gb allocation diff --git a/test/hotspot/jtreg/gc/shenandoah/TestWithLogLevel.java b/test/hotspot/jtreg/gc/shenandoah/TestWithLogLevel.java index b9214aeb53c64..f5650927c14a0 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestWithLogLevel.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestWithLogLevel.java @@ -23,7 +23,7 @@ */ /* - * @test + * @test id=default * @summary Test Shenandoah with different log levels * @requires vm.gc.Shenandoah * @@ -34,6 +34,17 @@ * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xms256M -Xmx1G -Xlog:gc*=trace TestWithLogLevel */ +/* + * @test id=generational + * @summary Test Shenandoah with different log levels + * @requires vm.gc.Shenandoah + * + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xms256M -Xmx1G -Xlog:gc*=error TestWithLogLevel + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xms256M -Xmx1G -Xlog:gc*=warning TestWithLogLevel + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xms256M -Xmx1G -Xlog:gc*=info TestWithLogLevel + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xms256M -Xmx1G -Xlog:gc*=debug TestWithLogLevel + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xms256M -Xmx1G -Xlog:gc*=trace TestWithLogLevel + */ import java.util.*; public class TestWithLogLevel { diff --git a/test/hotspot/jtreg/gc/shenandoah/TestWrongArrayMember.java b/test/hotspot/jtreg/gc/shenandoah/TestWrongArrayMember.java index ee645d994abf0..7045a85b2eccb 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestWrongArrayMember.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestWrongArrayMember.java @@ -26,8 +26,9 @@ * @test * @requires vm.gc.Shenandoah * - * @run main/othervm -Xmx128m -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC TestWrongArrayMember - * @run main/othervm -Xmx128m -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=iu TestWrongArrayMember + * @run main/othervm -Xmx128m -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC TestWrongArrayMember + * @run main/othervm -Xmx128m -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=iu TestWrongArrayMember + * @run main/othervm -Xmx128m -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational TestWrongArrayMember */ public class TestWrongArrayMember { diff --git a/test/hotspot/jtreg/gc/shenandoah/compiler/TestClone.java b/test/hotspot/jtreg/gc/shenandoah/compiler/TestClone.java index cb8582ee42094..4035a20c02d4e 100644 --- a/test/hotspot/jtreg/gc/shenandoah/compiler/TestClone.java +++ b/test/hotspot/jtreg/gc/shenandoah/compiler/TestClone.java @@ -205,7 +205,131 @@ * TestClone */ +/* + * @test id=generational + * @summary Test clone barriers work correctly + * @requires vm.gc.Shenandoah + * + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -Xms1g -Xmx1g + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * TestClone + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -Xms1g -Xmx1g + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * -Xint + * TestClone + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -Xms1g -Xmx1g + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * -XX:-TieredCompilation + * TestClone + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -Xms1g -Xmx1g + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * -XX:TieredStopAtLevel=1 + * TestClone + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -Xms1g -Xmx1g + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * -XX:TieredStopAtLevel=4 + * TestClone + */ + +/* + * @test id=generational-verify + * @summary Test clone barriers work correctly + * @requires vm.gc.Shenandoah + * + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -Xms1g -Xmx1g + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * -XX:+ShenandoahVerify + * TestClone + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -Xms1g -Xmx1g + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * -XX:+ShenandoahVerify + * -Xint + * TestClone + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -Xms1g -Xmx1g + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * -XX:+ShenandoahVerify + * -XX:-TieredCompilation + * TestClone + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -Xms1g -Xmx1g + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * -XX:+ShenandoahVerify + * -XX:TieredStopAtLevel=1 + * TestClone + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -Xms1g -Xmx1g + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * -XX:+ShenandoahVerify + * -XX:TieredStopAtLevel=4 + * TestClone + */ + + /* + * @test id=generational-no-coops + * @summary Test clone barriers work correctly + * @requires vm.gc.Shenandoah + * @requires vm.bits == "64" + * + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -Xms1g -Xmx1g + * -XX:-UseCompressedOops + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * TestClone + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -Xms1g -Xmx1g + * -XX:-UseCompressedOops + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * -Xint + * TestClone + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -Xms1g -Xmx1g + * -XX:-UseCompressedOops + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * -XX:-TieredCompilation + * TestClone + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -Xms1g -Xmx1g + * -XX:-UseCompressedOops + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * -XX:TieredStopAtLevel=1 + * TestClone + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -Xms1g -Xmx1g + * -XX:-UseCompressedOops + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * -XX:TieredStopAtLevel=4 + * TestClone + */ + /* + * @test id=generational-no-coops-verify + * @summary Test clone barriers work correctly + * @requires vm.gc.Shenandoah + * @requires vm.bits == "64" + * + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -Xms1g -Xmx1g + * -XX:-UseCompressedOops + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * -XX:+ShenandoahVerify + * TestClone + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -Xms1g -Xmx1g + * -XX:-UseCompressedOops + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * -XX:+ShenandoahVerify + * -Xint + * TestClone + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -Xms1g -Xmx1g + * -XX:-UseCompressedOops + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * -XX:+ShenandoahVerify + * -XX:-TieredCompilation + * TestClone + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -Xms1g -Xmx1g + * -XX:-UseCompressedOops + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * -XX:+ShenandoahVerify + * -XX:TieredStopAtLevel=1 + * TestClone + * @run main/othervm -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -Xms1g -Xmx1g + * -XX:-UseCompressedOops + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * -XX:+ShenandoahVerify + * -XX:TieredStopAtLevel=4 + * TestClone + */ public class TestClone { public static void main(String[] args) throws Exception { diff --git a/test/hotspot/jtreg/gc/shenandoah/compiler/TestReferenceCAS.java b/test/hotspot/jtreg/gc/shenandoah/compiler/TestReferenceCAS.java index ecfe46377b416..5ec6f95cb8645 100644 --- a/test/hotspot/jtreg/gc/shenandoah/compiler/TestReferenceCAS.java +++ b/test/hotspot/jtreg/gc/shenandoah/compiler/TestReferenceCAS.java @@ -53,6 +53,32 @@ * @run main/othervm -Diters=20000 -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:ShenandoahGCHeuristics=aggressive -XX:+UseShenandoahGC -XX:-UseCompressedOops -XX:TieredStopAtLevel=4 TestReferenceCAS */ +/* + * @test id=generational + * @summary Shenandoah reference CAS test + * @requires vm.gc.Shenandoah + * @modules java.base/jdk.internal.misc:+open + * + * @run main/othervm -Diters=20000 -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational TestReferenceCAS + * @run main/othervm -Diters=100 -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xint TestReferenceCAS + * @run main/othervm -Diters=20000 -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -XX:-TieredCompilation TestReferenceCAS + * @run main/othervm -Diters=20000 -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -XX:TieredStopAtLevel=1 TestReferenceCAS + * @run main/othervm -Diters=20000 -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -XX:TieredStopAtLevel=4 TestReferenceCAS + */ + +/* + * @test id=generational-no-coops + * @summary Shenandoah reference CAS test + * @requires vm.gc.Shenandoah + * @requires vm.bits == "64" + * @modules java.base/jdk.internal.misc:+open + * + * @run main/othervm -Diters=20000 -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -XX:-UseCompressedOops TestReferenceCAS + * @run main/othervm -Diters=100 -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -XX:-UseCompressedOops -Xint TestReferenceCAS + * @run main/othervm -Diters=20000 -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -XX:-UseCompressedOops -XX:-TieredCompilation TestReferenceCAS + * @run main/othervm -Diters=20000 -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -XX:-UseCompressedOops -XX:TieredStopAtLevel=1 TestReferenceCAS + * @run main/othervm -Diters=20000 -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -XX:-UseCompressedOops -XX:TieredStopAtLevel=4 TestReferenceCAS + */ import java.lang.reflect.Field; public class TestReferenceCAS { diff --git a/test/hotspot/jtreg/gc/shenandoah/generational/TestCLIModeGenerational.java b/test/hotspot/jtreg/gc/shenandoah/generational/TestCLIModeGenerational.java index 269b236a8dbd5..dfb62d680ae82 100644 --- a/test/hotspot/jtreg/gc/shenandoah/generational/TestCLIModeGenerational.java +++ b/test/hotspot/jtreg/gc/shenandoah/generational/TestCLIModeGenerational.java @@ -23,15 +23,15 @@ package gc.shenandoah.generational; -import sun.hotspot.WhiteBox; +import jdk.test.whitebox.WhiteBox; /* * @test TestCLIModeGenerational * @requires vm.gc.Shenandoah * @summary Test argument processing for -XX:+ShenandoahGCMode=generational. * @library /testlibrary /test/lib / - * @build sun.hotspot.WhiteBox - * @run driver jdk.test.lib.helpers.ClassFileInstaller sun.hotspot.WhiteBox + * @build jdk.test.whitebox.WhiteBox + * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox * @run main/othervm -Xbootclasspath/a:. * -XX:+IgnoreUnrecognizedVMOptions * -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI diff --git a/test/hotspot/jtreg/gc/shenandoah/generational/TestConcurrentEvac.java b/test/hotspot/jtreg/gc/shenandoah/generational/TestConcurrentEvac.java index 56fc01d87f1e5..4111c5eac5428 100644 --- a/test/hotspot/jtreg/gc/shenandoah/generational/TestConcurrentEvac.java +++ b/test/hotspot/jtreg/gc/shenandoah/generational/TestConcurrentEvac.java @@ -24,7 +24,7 @@ package gc.shenandoah.generational; -import sun.hotspot.WhiteBox; +import jdk.test.whitebox.WhiteBox; import java.util.Random; import java.util.HashMap; @@ -39,8 +39,8 @@ * @requires vm.gc.Shenandoah * @summary Confirm that card marking and remembered set scanning do not crash. * @library /testlibrary /test/lib / - * @build sun.hotspot.WhiteBox - * @run driver jdk.test.lib.helpers.ClassFileInstaller sun.hotspot.WhiteBox + * @build jdk.test.whitebox.WhiteBox + * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox * @run main/othervm -Xbootclasspath/a:. * -Xms256m -Xmx256m * -XX:+IgnoreUnrecognizedVMOptions diff --git a/test/hotspot/jtreg/gc/shenandoah/generational/TestSimpleGenerational.java b/test/hotspot/jtreg/gc/shenandoah/generational/TestSimpleGenerational.java index 53f61fd4361bf..38fe2e9f1a994 100644 --- a/test/hotspot/jtreg/gc/shenandoah/generational/TestSimpleGenerational.java +++ b/test/hotspot/jtreg/gc/shenandoah/generational/TestSimpleGenerational.java @@ -24,7 +24,7 @@ package gc.shenandoah.generational; -import sun.hotspot.WhiteBox; +import jdk.test.whitebox.WhiteBox; import java.util.Random; /* @@ -32,8 +32,8 @@ * @requires vm.gc.Shenandoah * @summary Confirm that card marking and remembered set scanning do not crash. * @library /testlibrary /test/lib / - * @build sun.hotspot.WhiteBox - * @run driver jdk.test.lib.helpers.ClassFileInstaller sun.hotspot.WhiteBox + * @build jdk.test.whitebox.WhiteBox + * @run driver jdk.test.lib.helpers.ClassFileInstaller jdk.test.whitebox.WhiteBox * @run main/othervm -Xbootclasspath/a:. * -XX:+IgnoreUnrecognizedVMOptions * -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI diff --git a/test/hotspot/jtreg/gc/shenandoah/jni/TestJNICritical.java b/test/hotspot/jtreg/gc/shenandoah/jni/TestJNICritical.java index c5bcc623d4535..93eee9fbe9de8 100644 --- a/test/hotspot/jtreg/gc/shenandoah/jni/TestJNICritical.java +++ b/test/hotspot/jtreg/gc/shenandoah/jni/TestJNICritical.java @@ -22,7 +22,7 @@ * */ -/* @test +/* @test id=default * @summary test JNI critical arrays support in Shenandoah * @key randomness * @requires vm.gc.Shenandoah @@ -32,6 +32,16 @@ * @run main/othervm/native -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCHeuristics=aggressive TestJNICritical */ + /* @test id=generational + * @summary test JNI critical arrays support in Shenandoah + * @key randomness + * @requires vm.gc.Shenandoah + * @library /test/lib + * + * @run main/othervm/native -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -XX:+ShenandoahVerify TestJNICritical + * @run main/othervm/native -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational TestJNICritical + */ + import java.util.Arrays; import java.util.Random; import jdk.test.lib.Utils; diff --git a/test/hotspot/jtreg/gc/shenandoah/jni/TestJNIGlobalRefs.java b/test/hotspot/jtreg/gc/shenandoah/jni/TestJNIGlobalRefs.java index 64520391f255a..bd8b0bae92a6b 100644 --- a/test/hotspot/jtreg/gc/shenandoah/jni/TestJNIGlobalRefs.java +++ b/test/hotspot/jtreg/gc/shenandoah/jni/TestJNIGlobalRefs.java @@ -41,6 +41,25 @@ * TestJNIGlobalRefs */ +/* @test id=generational-verify + * @summary Test JNI Global Refs with Shenandoah + * @requires vm.gc.Shenandoah + * + * @run main/othervm/native -Xmx1g -Xlog:gc -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * -XX:+ShenandoahVerify + * TestJNIGlobalRefs + */ + +/* @test id=generational + * @summary Test JNI Global Refs with Shenandoah + * @requires vm.gc.Shenandoah + * + * @run main/othervm/native -Xmx1g -Xlog:gc -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * TestJNIGlobalRefs + */ + import java.util.Arrays; import java.util.Random; diff --git a/test/hotspot/jtreg/gc/shenandoah/jni/TestPinnedGarbage.java b/test/hotspot/jtreg/gc/shenandoah/jni/TestPinnedGarbage.java index a6b53a0ee2b6b..ad2838e940d66 100644 --- a/test/hotspot/jtreg/gc/shenandoah/jni/TestPinnedGarbage.java +++ b/test/hotspot/jtreg/gc/shenandoah/jni/TestPinnedGarbage.java @@ -61,6 +61,22 @@ * TestPinnedGarbage */ +/* @test id=generational + * @summary Test that garbage in the pinned region does not crash VM + * @key randomness + * @requires vm.gc.Shenandoah + * @library /test/lib + * + * @run main/othervm/native -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -Xmx128m + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * TestPinnedGarbage + * + * @run main/othervm/native -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions -Xmx128m + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * -XX:+ShenandoahVerify + * TestPinnedGarbage + */ + import java.util.Arrays; import java.util.Random; import jdk.test.lib.Utils; diff --git a/test/hotspot/jtreg/gc/shenandoah/jvmti/TestHeapDump.java b/test/hotspot/jtreg/gc/shenandoah/jvmti/TestHeapDump.java index b570dad875feb..ed0d589f2fb0f 100644 --- a/test/hotspot/jtreg/gc/shenandoah/jvmti/TestHeapDump.java +++ b/test/hotspot/jtreg/gc/shenandoah/jvmti/TestHeapDump.java @@ -63,6 +63,46 @@ * -XX:+UseStringDeduplication TestHeapDump */ +/** + * @test id=generational + * @summary Tests JVMTI heap dumps + * @requires vm.gc.Shenandoah + * @requires vm.jvmti + * @compile TestHeapDump.java + * @run main/othervm/native/timeout=300 -agentlib:TestHeapDump + * -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * -XX:+UseShenandoahGC -Xmx128m + * -XX:ShenandoahGCMode=generational + * TestHeapDump + * + */ + +/** + * @test id=no-coops-generational + * @summary Tests JVMTI heap dumps + * @requires vm.gc.Shenandoah + * @requires vm.jvmti + * @requires vm.bits == "64" + * @compile TestHeapDump.java + * @run main/othervm/native/timeout=300 -agentlib:TestHeapDump + * -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * -XX:+UseShenandoahGC -Xmx128m + * -XX:ShenandoahGCMode=generational + * -XX:-UseCompressedOops TestHeapDump + */ + +/** + * @test id=generational-strdedup + * @summary Tests JVMTI heap dumps + * @requires vm.gc.Shenandoah + * @requires vm.jvmti + * @compile TestHeapDump.java + * @run main/othervm/native/timeout=300 -agentlib:TestHeapDump + * -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * -XX:+UseShenandoahGC -Xmx128m + * -XX:ShenandoahGCMode=generational + * -XX:+UseStringDeduplication TestHeapDump + */ import java.lang.ref.Reference; public class TestHeapDump { diff --git a/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestChurnNotifications.java b/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestChurnNotifications.java index e83b85c62b853..a42e41d6295db 100644 --- a/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestChurnNotifications.java +++ b/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestChurnNotifications.java @@ -104,6 +104,18 @@ * TestChurnNotifications */ +/* + * @test id=generational + * @summary Check that MX notifications are reported for all cycles + * @library /test/lib / + * @requires vm.gc.Shenandoah + * + * @run main/othervm -Xmx128m -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * -Dprecise=false + * TestChurnNotifications + */ + import java.util.*; import java.util.concurrent.atomic.*; import javax.management.*; @@ -128,6 +140,17 @@ public class TestChurnNotifications { static volatile Object sink; + private static final String DEFAULT_POOL_NAME = "Shenandoah"; + private static final String YOUNG_GEN_POOL_NAME = "Shenandoah Young Gen"; + + private static MemoryUsage getUsage(Map pools) { + MemoryUsage usage = pools.get(DEFAULT_POOL_NAME); + if (usage == null) { + usage = pools.get(YOUNG_GEN_POOL_NAME); + } + return usage; + } + public static void main(String[] args) throws Exception { final long startTime = System.currentTimeMillis(); @@ -141,8 +164,8 @@ public void handleNotification(Notification n, Object o) { Map mapBefore = info.getGcInfo().getMemoryUsageBeforeGc(); Map mapAfter = info.getGcInfo().getMemoryUsageAfterGc(); - MemoryUsage before = mapBefore.get("Shenandoah"); - MemoryUsage after = mapAfter.get("Shenandoah"); + MemoryUsage before = getUsage(mapBefore); + MemoryUsage after = getUsage(mapAfter); if ((before != null) && (after != null)) { long diff = before.getUsed() - after.getUsed(); diff --git a/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestMemoryMXBeans.java b/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestMemoryMXBeans.java index 56988e2f99340..63746564a6af5 100644 --- a/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestMemoryMXBeans.java +++ b/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestMemoryMXBeans.java @@ -23,7 +23,7 @@ */ /** - * @test + * @test id=default * @summary Test JMX memory beans * @requires vm.gc.Shenandoah * @modules java.base/jdk.internal.misc @@ -35,6 +35,19 @@ * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xms128m -Xmx1g -XX:ShenandoahUncommitDelay=0 TestMemoryMXBeans 128 1024 */ +/** + * @test id=generational + * @summary Test JMX memory beans + * @requires vm.gc.Shenandoah + * @modules java.base/jdk.internal.misc + * java.management + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xmx1g TestMemoryMXBeans -1 1024 + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xms1g -Xmx1g TestMemoryMXBeans 1024 1024 + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xms128m -Xmx1g TestMemoryMXBeans 128 1024 + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xms1g -Xmx1g -XX:ShenandoahUncommitDelay=0 TestMemoryMXBeans 1024 1024 + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xms128m -Xmx1g -XX:ShenandoahUncommitDelay=0 TestMemoryMXBeans 128 1024 + */ + import java.lang.management.*; import java.util.*; diff --git a/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestMemoryPools.java b/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestMemoryPools.java index 7c7cbe673845f..b6ec62de3678c 100644 --- a/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestMemoryPools.java +++ b/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestMemoryPools.java @@ -23,7 +23,7 @@ */ /** - * @test + * @test id=default * @summary Test JMX memory pools * @requires vm.gc.Shenandoah * @modules java.base/jdk.internal.misc @@ -31,6 +31,15 @@ * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -Xmx1g -Xms1g TestMemoryPools */ +/** + * @test id=generational + * @summary Test JMX memory pools + * @requires vm.gc.Shenandoah + * @modules java.base/jdk.internal.misc + * java.management + * @run main/othervm -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xmx1g -Xms1g TestMemoryPools + */ + import java.lang.management.*; import java.util.*; diff --git a/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestPauseNotifications.java b/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestPauseNotifications.java index b35e6eaf7b581..845c181d9d05c 100644 --- a/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestPauseNotifications.java +++ b/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestPauseNotifications.java @@ -98,6 +98,17 @@ * TestPauseNotifications */ +/* + * @test id=generational + * @summary Check that MX notifications are reported for all cycles + * @library /test/lib / + * @requires vm.gc.Shenandoah + * + * @run main/othervm -Xmx128m -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * TestPauseNotifications + */ + import java.util.*; import java.util.concurrent.atomic.*; import javax.management.*; diff --git a/test/hotspot/jtreg/gc/shenandoah/oom/TestAllocLargeObj.java b/test/hotspot/jtreg/gc/shenandoah/oom/TestAllocLargeObj.java deleted file mode 100644 index 3996c19a98d61..0000000000000 --- a/test/hotspot/jtreg/gc/shenandoah/oom/TestAllocLargeObj.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright (c) 2018, Red Hat, Inc. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -/** - * @test - * @summary Test allocation of small object to result OOM, but not to crash JVM - * @requires vm.gc.Shenandoah - * @library /test/lib - * @run driver TestAllocLargeObj - */ - -import jdk.test.lib.process.OutputAnalyzer; -import jdk.test.lib.process.ProcessTools; - -public class TestAllocLargeObj { - - static final int SIZE = 1 * 1024 * 1024; - static final int COUNT = 16; - - static volatile Object sink; - - public static void work() throws Exception { - Object[] root = new Object[COUNT]; - sink = root; - for (int c = 0; c < COUNT; c++) { - root[c] = new Object[SIZE]; - } - } - - public static void main(String[] args) throws Exception { - if (args.length > 0) { - work(); - return; - } - - { - ProcessBuilder pb = ProcessTools.createJavaProcessBuilder( - "-Xmx16m", - "-XX:+UnlockExperimentalVMOptions", - "-XX:+UseShenandoahGC", - TestAllocLargeObj.class.getName(), - "test"); - - OutputAnalyzer analyzer = new OutputAnalyzer(pb.start()); - analyzer.shouldHaveExitValue(1); - analyzer.shouldContain("java.lang.OutOfMemoryError: Java heap space"); - } - - { - ProcessBuilder pb = ProcessTools.createJavaProcessBuilder( - "-Xmx1g", - "-XX:+UnlockExperimentalVMOptions", - "-XX:+UseShenandoahGC", - TestAllocLargeObj.class.getName(), - "test"); - - OutputAnalyzer analyzer = new OutputAnalyzer(pb.start()); - analyzer.shouldHaveExitValue(0); - analyzer.shouldNotContain("java.lang.OutOfMemoryError: Java heap space"); - } - } -} diff --git a/test/hotspot/jtreg/gc/shenandoah/oom/TestAllocLargerThanHeap.java b/test/hotspot/jtreg/gc/shenandoah/oom/TestAllocLargerThanHeap.java deleted file mode 100644 index c996bfd714eca..0000000000000 --- a/test/hotspot/jtreg/gc/shenandoah/oom/TestAllocLargerThanHeap.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (c) 2018, Red Hat, Inc. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -/** - * @test - * @summary Test that allocation of the object larger than heap fails predictably - * @requires vm.gc.Shenandoah - * @library /test/lib - * @run driver TestAllocLargerThanHeap - */ - -import jdk.test.lib.process.OutputAnalyzer; -import jdk.test.lib.process.ProcessTools; - -public class TestAllocLargerThanHeap { - - static final int SIZE = 16 * 1024 * 1024; - - static volatile Object sink; - - public static void work() throws Exception { - sink = new Object[SIZE]; - } - - public static void main(String[] args) throws Exception { - if (args.length > 0) { - work(); - return; - } - - { - ProcessBuilder pb = ProcessTools.createJavaProcessBuilder( - "-Xmx16m", - "-XX:+UnlockExperimentalVMOptions", - "-XX:+UseShenandoahGC", - TestAllocLargerThanHeap.class.getName(), - "test"); - - OutputAnalyzer analyzer = new OutputAnalyzer(pb.start()); - analyzer.shouldHaveExitValue(1); - analyzer.shouldContain("java.lang.OutOfMemoryError: Java heap space"); - } - - { - ProcessBuilder pb = ProcessTools.createJavaProcessBuilder( - "-Xmx1g", - "-XX:+UnlockExperimentalVMOptions", - "-XX:+UseShenandoahGC", - TestAllocLargerThanHeap.class.getName(), - "test"); - - OutputAnalyzer analyzer = new OutputAnalyzer(pb.start()); - analyzer.shouldHaveExitValue(0); - analyzer.shouldNotContain("java.lang.OutOfMemoryError: Java heap space"); - } - } -} diff --git a/test/hotspot/jtreg/gc/shenandoah/oom/TestAllocOutOfMemory.java b/test/hotspot/jtreg/gc/shenandoah/oom/TestAllocOutOfMemory.java new file mode 100644 index 0000000000000..7a7502f2c3d16 --- /dev/null +++ b/test/hotspot/jtreg/gc/shenandoah/oom/TestAllocOutOfMemory.java @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2018, Red Hat, Inc. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +/** + * @test id=large + * @summary Test allocation of large objects results in OOM, but will not crash the JVM + * @requires vm.gc.Shenandoah + * @library /test/lib + * @run driver TestAllocOutOfMemory large + */ + +/** + * @test id=heap + * @summary Test allocation of a heap-sized object results in OOM, but will not crash the JVM + * @requires vm.gc.Shenandoah + * @library /test/lib + * @run driver TestAllocOutOfMemory heap + */ + +/** + * @test id=small + * @summary Test allocation of small objects results in OOM, but will not crash the JVM + * @requires vm.gc.Shenandoah + * @library /test/lib + * @run driver TestAllocOutOfMemory small + */ +import jdk.test.lib.process.OutputAnalyzer; +import jdk.test.lib.process.ProcessTools; + +public class TestAllocOutOfMemory { + + static volatile Object sink; + + public static void work(int size, int count) throws Exception { + Object[] root = new Object[count]; + sink = root; + for (int c = 0; c < count; c++) { + root[c] = new Object[size]; + } + } + + private static void allocate(String size) throws Exception { + switch (size) { + case "large": + work(1024 * 1024, 16); + break; + case "heap": + work(16 * 1024 * 1024, 1); + break; + case "small": + work(1, 16 * 1024 * 1024); + break; + default: + throw new IllegalArgumentException("Usage: test [large|small|heap]"); + } + } + + public static void main(String[] args) throws Exception { + if (args.length > 1) { + // Called from test, size is second argument + String size = args[1]; + allocate(size); + return; + } + + // Called from jtreg, size is first argument + String size = args[0]; + { + expectFailure("-Xmx16m", + "-XX:+UnlockExperimentalVMOptions", + "-XX:+UseShenandoahGC", + TestAllocOutOfMemory.class.getName(), + "test", size); + + expectFailure("-Xmx16m", + "-XX:+UnlockExperimentalVMOptions", + "-XX:+UseShenandoahGC", "-XX:ShenandoahGCMode=generational", + TestAllocOutOfMemory.class.getName(), + "test", size); + } + + { + expectSuccess("-Xmx1g", + "-XX:+UnlockExperimentalVMOptions", + "-XX:+UseShenandoahGC", + TestAllocOutOfMemory.class.getName(), + "test", size); + + expectSuccess("-Xmx1g", + "-XX:+UnlockExperimentalVMOptions", + "-XX:+UseShenandoahGC", "-XX:ShenandoahGCMode=generational", + TestAllocOutOfMemory.class.getName(), + "test", size); + } + } + + private static void expectSuccess(String... args) throws Exception { + ProcessBuilder pb = ProcessTools.createJavaProcessBuilder(args); + OutputAnalyzer analyzer = new OutputAnalyzer(pb.start()); + analyzer.shouldHaveExitValue(0); + analyzer.shouldNotContain("java.lang.OutOfMemoryError: Java heap space"); + } + + private static void expectFailure(String... args) throws Exception { + ProcessBuilder pb = ProcessTools.createJavaProcessBuilder(args); + OutputAnalyzer analyzer = new OutputAnalyzer(pb.start()); + analyzer.shouldHaveExitValue(1); + analyzer.shouldContain("java.lang.OutOfMemoryError: Java heap space"); + } +} diff --git a/test/hotspot/jtreg/gc/shenandoah/oom/TestAllocSmallObj.java b/test/hotspot/jtreg/gc/shenandoah/oom/TestAllocSmallObj.java deleted file mode 100644 index 0820237ab9421..0000000000000 --- a/test/hotspot/jtreg/gc/shenandoah/oom/TestAllocSmallObj.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (c) 2018, Red Hat, Inc. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - * - */ - -/** - * @test - * @summary Test allocation of small object to result OOM, but not to crash JVM - * @requires vm.gc.Shenandoah - * @library /test/lib - * @run driver TestAllocSmallObj - */ - -import jdk.test.lib.process.OutputAnalyzer; -import jdk.test.lib.process.ProcessTools; - -public class TestAllocSmallObj { - - static final int COUNT = 16 * 1024 * 1024; - - static volatile Object sink; - - public static void work() throws Exception { - Object[] root = new Object[COUNT]; - sink = root; - for (int c = 0; c < COUNT; c++) { - root[c] = new Object(); - } - } - - public static void main(String[] args) throws Exception { - if (args.length > 0) { - work(); - return; - } - - { - ProcessBuilder pb = ProcessTools.createJavaProcessBuilder( - "-Xmx16m", - "-XX:+UnlockExperimentalVMOptions", - "-XX:+UseShenandoahGC", - TestAllocSmallObj.class.getName(), - "test"); - - OutputAnalyzer analyzer = new OutputAnalyzer(pb.start()); - analyzer.shouldHaveExitValue(1); - analyzer.shouldContain("java.lang.OutOfMemoryError: Java heap space"); - } - - { - ProcessBuilder pb = ProcessTools.createJavaProcessBuilder( - "-Xmx1g", - "-XX:+UnlockExperimentalVMOptions", - "-XX:+UseShenandoahGC", - TestAllocSmallObj.class.getName(), - "test"); - - OutputAnalyzer analyzer = new OutputAnalyzer(pb.start()); - analyzer.shouldHaveExitValue(0); - analyzer.shouldNotContain("java.lang.OutOfMemoryError: Java heap space"); - } - } -} diff --git a/test/hotspot/jtreg/gc/shenandoah/oom/TestClassLoaderLeak.java b/test/hotspot/jtreg/gc/shenandoah/oom/TestClassLoaderLeak.java index 21962b2182d3e..62315af2c0524 100644 --- a/test/hotspot/jtreg/gc/shenandoah/oom/TestClassLoaderLeak.java +++ b/test/hotspot/jtreg/gc/shenandoah/oom/TestClassLoaderLeak.java @@ -125,9 +125,10 @@ public static void main(String[] args) throws Exception { } String[][][] modeHeuristics = new String[][][] { - {{"satb"}, {"adaptive", "compact", "static", "aggressive"}}, - {{"iu"}, {"adaptive", "aggressive"}}, - {{"passive"}, {"passive"}} + {{"satb"}, {"adaptive", "compact", "static", "aggressive"}}, + {{"iu"}, {"adaptive", "aggressive"}}, + {{"passive"}, {"passive"}}, + {{"generational"}, {"adaptive"}} }; for (String[][] mh : modeHeuristics) { diff --git a/test/hotspot/jtreg/gc/shenandoah/oom/TestThreadFailure.java b/test/hotspot/jtreg/gc/shenandoah/oom/TestThreadFailure.java index a342b4160104b..37ce09d1a2fcb 100644 --- a/test/hotspot/jtreg/gc/shenandoah/oom/TestThreadFailure.java +++ b/test/hotspot/jtreg/gc/shenandoah/oom/TestThreadFailure.java @@ -74,5 +74,19 @@ public static void main(String[] args) throws Exception { analyzer.shouldContain("java.lang.OutOfMemoryError"); analyzer.shouldContain("All good"); } + + { + ProcessBuilder pb = ProcessTools.createJavaProcessBuilder( + "-Xmx32m", + "-XX:+UnlockExperimentalVMOptions", + "-XX:+UseShenandoahGC", "-XX:ShenandoahGCMode=generational", + TestThreadFailure.class.getName(), + "test"); + + OutputAnalyzer analyzer = new OutputAnalyzer(pb.start()); + analyzer.shouldHaveExitValue(0); + analyzer.shouldContain("java.lang.OutOfMemoryError"); + analyzer.shouldContain("All good"); + } } } diff --git a/test/hotspot/jtreg/gc/shenandoah/options/TestModeUnlock.java b/test/hotspot/jtreg/gc/shenandoah/options/TestModeUnlock.java index 6c03281c3ca2e..7b72e26929217 100644 --- a/test/hotspot/jtreg/gc/shenandoah/options/TestModeUnlock.java +++ b/test/hotspot/jtreg/gc/shenandoah/options/TestModeUnlock.java @@ -47,6 +47,7 @@ public static void main(String[] args) throws Exception { testWith("-XX:ShenandoahGCMode=satb", Mode.PRODUCT); testWith("-XX:ShenandoahGCMode=iu", Mode.EXPERIMENTAL); testWith("-XX:ShenandoahGCMode=passive", Mode.DIAGNOSTIC); + testWith("-XX:ShenandoahGCMode=generational", Mode.PRODUCT); } private static void testWith(String h, Mode mode) throws Exception { diff --git a/test/hotspot/jtreg/gc/stress/gcbasher/TestGCBasherWithShenandoah.java b/test/hotspot/jtreg/gc/stress/gcbasher/TestGCBasherWithShenandoah.java index 6e98f99b54823..3dc34c90584a8 100644 --- a/test/hotspot/jtreg/gc/stress/gcbasher/TestGCBasherWithShenandoah.java +++ b/test/hotspot/jtreg/gc/stress/gcbasher/TestGCBasherWithShenandoah.java @@ -267,6 +267,43 @@ * gc.stress.gcbasher.TestGCBasherWithShenandoah 120000 */ +/* + * @test id=generational + * @key stress + * @library / + * @requires vm.gc.Shenandoah + * @requires vm.flavor == "server" & !vm.emulatedClient + * @summary Stress the Shenandoah GC by trying to make old objects more likely to be garbage than young objects. + * + * @run main/othervm/timeout=200 -Xlog:gc*=info -Xmx1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * -XX:+ShenandoahVerify + * gc.stress.gcbasher.TestGCBasherWithShenandoah 120000 + * + * @run main/othervm/timeout=200 -Xlog:gc*=info -Xmx1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * gc.stress.gcbasher.TestGCBasherWithShenandoah 120000 + */ + + /* + * @test id=generational-deopt-nmethod + * @key stress + * @library / + * @requires vm.gc.Shenandoah + * @requires vm.flavor == "server" & !vm.emulatedClient & vm.opt.ClassUnloading != false + * @summary Stress Shenandoah GC with nmethod barrier forced deoptimization enabled. + * + * @run main/othervm/timeout=200 -Xlog:gc*=info,nmethod+barrier=trace -Xmx1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * -XX:+DeoptimizeNMethodBarriersALot -XX:-Inline + * -XX:+ShenandoahVerify + * gc.stress.gcbasher.TestGCBasherWithShenandoah 120000 + * + * @run main/othervm/timeout=200 -Xlog:gc*=info,nmethod+barrier=trace -Xmx1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * -XX:+DeoptimizeNMethodBarriersALot -XX:-Inline + * gc.stress.gcbasher.TestGCBasherWithShenandoah 120000 + */ public class TestGCBasherWithShenandoah { public static void main(String[] args) throws IOException { diff --git a/test/hotspot/jtreg/gc/stress/gclocker/TestGCLockerWithShenandoah.java b/test/hotspot/jtreg/gc/stress/gclocker/TestGCLockerWithShenandoah.java index 586fdc798817f..326b2f70b2e91 100644 --- a/test/hotspot/jtreg/gc/stress/gclocker/TestGCLockerWithShenandoah.java +++ b/test/hotspot/jtreg/gc/stress/gclocker/TestGCLockerWithShenandoah.java @@ -40,6 +40,24 @@ * gc.stress.gclocker.TestGCLockerWithShenandoah */ +/* + * @test id=generational + * @library / + * @requires vm.gc.Shenandoah + * @summary Stress Shenandoah's JNI handling by calling GetPrimitiveArrayCritical while concurrently filling up old gen. + * + * @run main/native/othervm/timeout=200 -Xlog:gc*=info -Xms1500m -Xmx1500m -XX:+UnlockExperimentalVMOptions -XX:+UnlockDiagnosticVMOptions + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * -XX:ShenandoahMaxYoungPercentage=90 + * -XX:+ShenandoahVerify + * gc.stress.gclocker.TestGCLockerWithShenandoah + * + * @run main/native/othervm/timeout=200 -Xlog:gc*=info -Xms1500m -Xmx1500m -XX:+UnlockExperimentalVMOptions -XX:+UnlockDiagnosticVMOptions + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * -XX:ShenandoahMaxYoungPercentage=90 + * gc.stress.gclocker.TestGCLockerWithShenandoah + */ + /* * @test id=aggressive * @library / diff --git a/test/hotspot/jtreg/gc/stress/gcold/TestGCOldWithShenandoah.java b/test/hotspot/jtreg/gc/stress/gcold/TestGCOldWithShenandoah.java index f1e34bb1b0b66..cf8ea24b13644 100644 --- a/test/hotspot/jtreg/gc/stress/gcold/TestGCOldWithShenandoah.java +++ b/test/hotspot/jtreg/gc/stress/gcold/TestGCOldWithShenandoah.java @@ -141,6 +141,22 @@ * gc.stress.gcold.TestGCOld 50 1 20 10 10000 */ +/* + * @test id=generational + * @key stress randomness + * @library / /test/lib + * @requires vm.gc.Shenandoah + * @summary Stress the GC by trying to make old objects more likely to be garbage than young objects. + * + * @run main/othervm/timeout=600 -Xmx384M -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * -XX:+ShenandoahVerify + * gc.stress.gcold.TestGCOld 50 1 20 10 10000 + * + * @run main/othervm -Xmx384M -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * gc.stress.gcold.TestGCOld 50 1 20 10 10000 + */ public class TestGCOldWithShenandoah { public static void main(String[] args) { diff --git a/test/hotspot/jtreg/gc/stress/jfr/TestStressBigAllocationGCEventsWithShenandoah.java b/test/hotspot/jtreg/gc/stress/jfr/TestStressBigAllocationGCEventsWithShenandoah.java new file mode 100644 index 0000000000000..7c7e78378d388 --- /dev/null +++ b/test/hotspot/jtreg/gc/stress/jfr/TestStressBigAllocationGCEventsWithShenandoah.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.jfr.event.gc.detailed; + +/** + * @test id=default + * @key randomness + * @requires vm.hasJFR + * @requires vm.gc == "Shenandoah" + * @library /test/lib /test/jdk + * @run main/othervm -XX:+UseShenandoahGC -Xmx256m jdk.jfr.event.gc.detailed.TestStressBigAllocationGCEventsWithShenandoah 1048576 + */ + + /** + * @test id=generational + * @key randomness + * @requires vm.hasJFR + * @requires vm.gc == "Shenandoah" + * @library /test/lib /test/jdk + * @run main/othervm -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational -Xmx256m jdk.jfr.event.gc.detailed.TestStressBigAllocationGCEventsWithShenandoah 1048576 + */ +public class TestStressBigAllocationGCEventsWithShenandoah { + + public static void main(String[] args) throws Exception { + new StressAllocationGCEvents().run(args); + } +} diff --git a/test/hotspot/jtreg/gc/stress/systemgc/TestSystemGCWithShenandoah.java b/test/hotspot/jtreg/gc/stress/systemgc/TestSystemGCWithShenandoah.java index 1b12e22b62f86..eed284e19f180 100644 --- a/test/hotspot/jtreg/gc/stress/systemgc/TestSystemGCWithShenandoah.java +++ b/test/hotspot/jtreg/gc/stress/systemgc/TestSystemGCWithShenandoah.java @@ -41,6 +41,23 @@ * gc.stress.systemgc.TestSystemGCWithShenandoah 270 */ +/* + * @test id=generational + * @key stress + * @library / + * @requires vm.gc.Shenandoah + * @summary Stress the Shenandoah GC full GC by allocating objects of different lifetimes concurrently with System.gc(). + * + * @run main/othervm/timeout=300 -Xlog:gc*=info -Xmx512m -XX:+UnlockExperimentalVMOptions -XX:+UnlockDiagnosticVMOptions + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * -XX:+ShenandoahVerify + * gc.stress.systemgc.TestSystemGCWithShenandoah 270 + * + * @run main/othervm/timeout=300 -Xlog:gc*=info -Xmx512m -XX:+UnlockExperimentalVMOptions -XX:+UnlockDiagnosticVMOptions + * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational + * gc.stress.systemgc.TestSystemGCWithShenandoah 270 + */ + /* * @test id=iu * @key stress From f7cede97c507cf847579f001de311353bd3c7d86 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Thu, 30 Mar 2023 22:36:46 +0000 Subject: [PATCH 200/254] Recompute generation accounts after fullgc Reviewed-by: ysr, wkemper --- .../share/gc/shenandoah/shenandoahFullGC.cpp | 60 +++++++++++++++++++ .../share/gc/shenandoah/shenandoahFullGC.hpp | 1 + .../gc/shenandoah/shenandoahGeneration.cpp | 7 +++ .../gc/shenandoah/shenandoahGeneration.hpp | 2 + .../gc/shenandoah/shenandoahPhaseTimings.hpp | 1 + 5 files changed, 71 insertions(+) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index ce2a80974c0fb..23fdbf7407854 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -322,6 +322,8 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) { phase3_update_references(); phase4_compact_objects(worker_slices); + + phase5_epilog(); } { @@ -1446,6 +1448,64 @@ void ShenandoahFullGC::phase4_compact_objects(ShenandoahHeapRegionSet** worker_s ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_humong); compact_humongous_objects(); } +} + +static void account_for_region(ShenandoahHeapRegion* r, size_t ®ion_count, size_t ®ion_usage, size_t &humongous_waste) { + region_count++; + region_usage += r->used(); + if (r->is_humongous_start()) { + // For each humongous object, we take this path once regardless of how many regions it spans. + HeapWord* obj_addr = r->bottom(); + oop obj = cast_to_oop(obj_addr); + size_t word_size = obj->size(); + size_t region_size_words = ShenandoahHeapRegion::region_size_words(); + size_t overreach = word_size % region_size_words; + if (overreach != 0) { + humongous_waste += (region_size_words - overreach) * HeapWordSize; + } + // else, this humongous object aligns exactly on region size, so no waste. + } +} + +void ShenandoahFullGC::phase5_epilog() { + GCTraceTime(Info, gc, phases) time("Phase 5: Full GC epilog", _gc_timer); + ShenandoahHeap* heap = ShenandoahHeap::heap(); + size_t num_regions = heap->num_regions(); + size_t young_usage = 0; + size_t young_regions = 0; + size_t young_humongous_waste = 0; + size_t old_usage = 0; + size_t old_regions = 0; + size_t old_humongous_waste = 0; + ShenandoahHeapRegion* r; + + if (heap->mode()->is_generational()) { + // TODO: We may be able remove code that recomputes generation usage after we fix the incremental updates to generation + // usage that are scattered throughout the existing Full GC implementation. There's an error in there somewhere that + // has not yet been figured out. Or maybe it is easier to just not try to do the generation accounting on the fly, keep + // this code, and remove all of the other attempts to increase/decrease affiliated regions, used, and humongous_waste. + { + ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_recompute_generation_usage); + for (size_t i = 0; i < num_regions; i++) { + switch (heap->region_affiliation(i)) { + case ShenandoahRegionAffiliation::FREE: + break; + case ShenandoahRegionAffiliation::YOUNG_GENERATION: + r = heap->get_region(i); + account_for_region(r, young_regions, young_usage, young_humongous_waste); + break; + case ShenandoahRegionAffiliation::OLD_GENERATION: + r = heap->get_region(i); + account_for_region(r, old_regions, old_usage, old_humongous_waste); + break; + default: + assert(false, "Should not reach"); + } + } + heap->old_generation()->establish_usage(old_regions, old_usage, old_humongous_waste); + heap->young_generation()->establish_usage(young_regions, young_usage, young_humongous_waste); + } + } // Reset complete bitmap. We're about to reset the complete-top-at-mark-start pointer // and must ensure the bitmap is in sync. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.hpp index 1c1653e59ec56..6687116b21f78 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.hpp @@ -81,6 +81,7 @@ class ShenandoahFullGC : public ShenandoahGC { void phase2_calculate_target_addresses(ShenandoahHeapRegionSet** worker_slices); void phase3_update_references(); void phase4_compact_objects(ShenandoahHeapRegionSet** worker_slices); + void phase5_epilog(); void distribute_slices(ShenandoahHeapRegionSet** worker_slices); void calculate_target_humongous_objects(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index 36dbb15620152..7356ab9f57679 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -943,6 +943,13 @@ size_t ShenandoahGeneration::decrement_affiliated_region_count() { return _affiliated_region_count; } +void ShenandoahGeneration::establish_usage(size_t num_regions, size_t num_bytes, size_t humongous_waste) { + assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at a safepoint"); + _affiliated_region_count = num_regions; + _used = num_bytes; + // future improvement: _humongous_waste = humongous_waste; +} + void ShenandoahGeneration::clear_used() { assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at a safepoint"); // Do this atomically to assure visibility to other threads, even though these other threads may be idle "right now".. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index fc4c6ea905e47..b12af084d112e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -183,6 +183,8 @@ class ShenandoahGeneration : public CHeapObj { // Return the updated value of affiliated_region_count size_t decrement_affiliated_region_count(); + void establish_usage(size_t num_regions, size_t num_bytes, size_t humongous_waste); + void clear_used(); void increase_used(size_t bytes); void decrease_used(size_t bytes); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp index 35ebf73327db5..d3d9f4d2b91fd 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp @@ -178,6 +178,7 @@ class outputStream; f(full_gc_copy_objects, " Copy Objects") \ f(full_gc_copy_objects_regular, " Regular Objects") \ f(full_gc_copy_objects_humong, " Humongous Objects") \ + f(full_gc_recompute_generation_usage, " Recompute generation usage") \ f(full_gc_copy_objects_reset_complete, " Reset Complete Bitmap") \ f(full_gc_copy_objects_rebuild, " Rebuild Region Sets") \ f(full_gc_reconstruct_remembered_set, " Reconstruct Remembered Set") \ From ba4161de7687db8ab225a0bbff52e56b1dbefbec Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Fri, 31 Mar 2023 17:47:26 +0000 Subject: [PATCH 201/254] Fix satb barrier for object array copy Reviewed-by: shade --- .../gc/shenandoah/shenandoahBarrierSet.hpp | 2 +- .../shenandoahBarrierSet.inline.hpp | 57 +++++++++++++++++-- 2 files changed, 52 insertions(+), 7 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp index c3cb1a8069203..16453e3275e87 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp @@ -122,7 +122,7 @@ class ShenandoahBarrierSet: public BarrierSet { private: template - inline void arraycopy_marking(T* src, T* dst, size_t count); + inline void arraycopy_marking(T* src, T* dst, size_t count, bool is_old_marking); template inline void arraycopy_evacuation(T* src, size_t count); template diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp index ada51264aad5f..a4bf8043ab7ac 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp @@ -405,7 +405,7 @@ void ShenandoahBarrierSet::arraycopy_barrier(T* src, T* dst, size_t count) { } int gc_state = _heap->gc_state(); if ((gc_state & ShenandoahHeap::YOUNG_MARKING) != 0) { - arraycopy_marking(src, dst, count); + arraycopy_marking(src, dst, count, false); return; } @@ -422,17 +422,62 @@ void ShenandoahBarrierSet::arraycopy_barrier(T* src, T* dst, size_t count) { // Note that we can't do the arraycopy marking using the 'src' array when // SATB mode is enabled (so we can't do this as part of the iteration for // evacuation or update references). - arraycopy_marking(src, dst, count); + arraycopy_marking(src, dst, count, true); } } } template -void ShenandoahBarrierSet::arraycopy_marking(T* src, T* dst, size_t count) { +void ShenandoahBarrierSet::arraycopy_marking(T* src, T* dst, size_t count, bool is_old_marking) { assert(_heap->is_concurrent_mark_in_progress(), "only during marking"); - T* array = ShenandoahSATBBarrier ? dst : src; - if (!_heap->marking_context()->allocated_after_mark_start(reinterpret_cast(array))) { - arraycopy_work(array, count); + /* + * Note that an old-gen object is considered live if it is live at the start of OLD marking or if it is promoted + * following the start of OLD marking. + * + * 1. Every object promoted following the start of OLD marking will be above TAMS within its old-gen region + * 2. Every object live at the start of OLD marking will be referenced from a "root" or it will be referenced from + * another live OLD-gen object. With regards to old-gen, roots include stack locations and all of live young-gen. + * All root references to old-gen are identified during a bootstrap young collection. All references from other + * old-gen objects will be marked during the traversal of all old objects, or will be marked by the SATB barrier. + * + * During old-gen marking (which is interleaved with young-gen collections), call arraycopy_work() if: + * + * 1. The overwritten array resides in old-gen and it is below TAMS within its old-gen region + * 2. Do not call arraycopy_work for any array residing in young-gen because young-gen collection is idle at this time + * + * During young-gen marking, call arraycopy_work() if: + * + * 1. The overwritten array resides in young-gen and is below TAMS within its young-gen region + * 2. Additionally, if array resides in old-gen, regardless of its relationship to TAMS because this old-gen array + * may hold references to young-gen + */ + if (ShenandoahSATBBarrier) { + T* array = dst; + HeapWord* array_addr = reinterpret_cast(array); + ShenandoahHeapRegion* r = _heap->heap_region_containing(array_addr); + if (is_old_marking) { + // Generational, old marking + assert(_heap->mode()->is_generational(), "Invariant"); + if (r->is_old() && (array_addr < _heap->marking_context()->top_at_mark_start(r))) { + arraycopy_work(array, count); + } + } else if (_heap->mode()->is_generational()) { + // Generational, young marking + if (r->is_old() || (array_addr < _heap->marking_context()->top_at_mark_start(r))) { + arraycopy_work(array, count); + } + } else if (array_addr < _heap->marking_context()->top_at_mark_start(r)) { + // Non-generational, marking + arraycopy_work(array, count); + } + } else { + // Incremental Update mode, marking + T* array = src; + HeapWord* array_addr = reinterpret_cast(array); + ShenandoahHeapRegion* r = _heap->heap_region_containing(array_addr); + if (array_addr < _heap->marking_context()->top_at_mark_start(r)) { + arraycopy_work(array, count); + } } } From f75b50b61e5e69e23ce2361ba4e22ef88060fd69 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Fri, 31 Mar 2023 19:20:04 +0000 Subject: [PATCH 202/254] Abandon mixed collections if all candidates are pinned Reviewed-by: kdnilsen, ysr --- .../heuristics/shenandoahOldHeuristics.cpp | 31 ++++++++- .../heuristics/shenandoahOldHeuristics.hpp | 1 + .../gc/shenandoah/shenandoahControlThread.cpp | 1 + .../gc/shenandoah/shenandoahGeneration.cpp | 3 +- .../share/gc/shenandoah/shenandoahHeap.cpp | 7 +- .../gc/shenandoah/shenandoahOldGeneration.cpp | 69 ++++++++++++------- .../gc/shenandoah/shenandoahOldGeneration.hpp | 6 +- .../gc/shenandoah/shenandoah_globals.hpp | 9 +++ .../test_shenandoahOldHeuristic.cpp | 24 +++++++ 9 files changed, 115 insertions(+), 36 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp index 32d2f02f39c6e..a4da2f460a63c 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp @@ -112,12 +112,39 @@ bool ShenandoahOldHeuristics::prime_collection_set(ShenandoahCollectionSet* coll } if (unprocessed_old_collection_candidates() == 0) { + // We have added the last of our collection candidates to a mixed collection. _old_generation->transition_to(ShenandoahOldGeneration::IDLE); + } else if (included_old_regions == 0) { + // We have candidates, but none were included for evacuation - are they all pinned? + // or did we just not have enough room for any of them in this collection set? + // We don't want a region with a stuck pin to prevent subsequent old collections, so + // if they are all pinned we transition to a state that will allow us to make these uncollected + // (pinned) regions parseable. + if (all_candidates_are_pinned()) { + log_info(gc)("All candidate regions " UINT32_FORMAT " are pinned.", unprocessed_old_collection_candidates()); + _old_generation->transition_to(ShenandoahOldGeneration::WAITING_FOR_FILL); + } } return (included_old_regions > 0); } +bool ShenandoahOldHeuristics::all_candidates_are_pinned() { +#ifdef ASSERT + if (uint(os::random()) % 100 < ShenandoahCoalesceChance) { + return true; + } +#endif + + for (uint i = _next_old_collection_candidate; i < _last_old_collection_candidate; ++i) { + auto region = _region_data[i]._region; + if (!region->is_pinned()) { + return false; + } + } + return true; +} + void ShenandoahOldHeuristics::slide_pinned_regions_to_front() { // Find the first unpinned region to the left of the next region that // will be added to the collection set. These regions will have been @@ -273,7 +300,7 @@ void ShenandoahOldHeuristics::prepare_for_old_collections() { if (unprocessed_old_collection_candidates() == 0) { _old_generation->transition_to(ShenandoahOldGeneration::IDLE); } else { - _old_generation->transition_to(ShenandoahOldGeneration::WAITING); + _old_generation->transition_to(ShenandoahOldGeneration::WAITING_FOR_EVAC); } } @@ -349,7 +376,7 @@ bool ShenandoahOldHeuristics::should_start_gc() { // // Future refinement: under certain circumstances, we might be more sophisticated about this choice. // For example, we could choose to abandon the previous old collection before it has completed evacuations. - if (unprocessed_old_collection_candidates() > 0) { + if (!_old_generation->can_start_gc()) { return false; } diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp index 09a49977e0f61..9eb065ef897b2 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp @@ -161,6 +161,7 @@ class ShenandoahOldHeuristics : public ShenandoahHeuristics { private: void slide_pinned_regions_to_front(); + bool all_candidates_are_pinned(); }; #endif // SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHOLDHEURISTICS_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index 8b09813ef402b..48b8e4b26856b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -507,6 +507,7 @@ void ShenandoahControlThread::service_concurrent_old_cycle(const ShenandoahHeap* TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); switch (original_state) { + case ShenandoahOldGeneration::WAITING_FOR_FILL: case ShenandoahOldGeneration::IDLE: { assert(!heap->is_concurrent_old_mark_in_progress(), "Old already in progress."); assert(old_generation->task_queues()->is_empty(), "Old mark queues should be empty."); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index 7356ab9f57679..cef6aed6e1ae1 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -249,9 +249,8 @@ void ShenandoahGeneration::compute_evacuation_budgets(ShenandoahHeap* heap, bool // accumulate there until they can be promoted. This increases the young-gen marking and evacuation work. // Do not fill up old-gen memory with promotions. Reserve some amount of memory for compaction purposes. - ShenandoahOldHeuristics* old_heuristics = heap->old_heuristics(); size_t young_evac_reserve_max = 0; - if (old_heuristics->unprocessed_old_collection_candidates() > 0) { + if (heap->doing_mixed_evacuations()) { // Compute old_evacuation_reserve: how much memory are we reserving to hold the results of // evacuating old-gen heap regions? In order to sustain a consistent pace of young-gen collections, // the goal is to maintain a consistent value for this parameter (when the candidate set is not diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 5aeee487a5980..7cb293d820dbe 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -678,7 +678,7 @@ ShenandoahOldHeuristics* ShenandoahHeap::old_heuristics() { } bool ShenandoahHeap::doing_mixed_evacuations() { - return old_heuristics()->unprocessed_old_collection_candidates() > 0; + return _old_generation->state() == ShenandoahOldGeneration::WAITING_FOR_EVAC; } bool ShenandoahHeap::is_old_bitmap_stable() const { @@ -1068,10 +1068,7 @@ void ShenandoahHeap::cancel_old_gc() { } bool ShenandoahHeap::is_old_gc_active() { - return is_concurrent_old_mark_in_progress() - || is_prepare_for_old_mark_in_progress() - || old_heuristics()->unprocessed_old_collection_candidates() > 0 - || young_generation()->old_gen_task_queues() != nullptr; + return _old_generation->state() != ShenandoahOldGeneration::IDLE; } void ShenandoahHeap::coalesce_and_fill_old_regions() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp index 3ade5f175ef80..ee54da2f4e595 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp @@ -255,6 +255,9 @@ bool ShenandoahOldGeneration::coalesce_and_fill() { uint nworkers = workers->active_workers(); log_debug(gc)("Starting (or resuming) coalesce-and-fill of old heap regions"); + // This code will see the same set of regions to fill on each resumption as it did + // on the initial run. That's okay because each region keeps track of its own coalesce + // and fill state. Regions that were filled on a prior attempt will not try to fill again. uint coalesce_and_fill_regions_count = old_heuristics->get_coalesce_and_fill_candidates(_coalesce_and_fill_region_array); assert(coalesce_and_fill_regions_count <= heap->num_regions(), "Sanity"); ShenandoahConcurrentCoalesceAndFillTask task(nworkers, _coalesce_and_fill_region_array, coalesce_and_fill_regions_count); @@ -263,6 +266,7 @@ bool ShenandoahOldGeneration::coalesce_and_fill() { if (task.is_completed()) { // Remember that we're done with coalesce-and-fill. heap->set_prepare_for_old_mark_in_progress(false); + old_heuristics->abandon_collection_candidates(); transition_to(BOOTSTRAPPING); return true; } else { @@ -323,7 +327,8 @@ const char* ShenandoahOldGeneration::state_name(State state) { case FILLING: return "Coalescing"; case BOOTSTRAPPING: return "Bootstrapping"; case MARKING: return "Marking"; - case WAITING: return "Waiting"; + case WAITING_FOR_EVAC: return "Waiting for evacuation"; + case WAITING_FOR_FILL: return "Waiting for fill"; default: ShouldNotReachHere(); return "Unknown"; @@ -358,46 +363,54 @@ void ShenandoahOldGeneration::transition_to(State new_state) { // | | v // | | +-----------------+ +--------------------+ // | | | FILLING | <-> | YOUNG GC | -// | | | | | (RSet Uses Bitmap) | -// | | +-----------------+ +--------------------+ -// | | | -// | | | Reset Bitmap -// | | v -// | | +-----------------+ -// | | | BOOTSTRAP | -// | | | | -// | | +-----------------+ -// | | | -// | | | Continue Marking -// | | v -// | | +-----------------+ +----------------------+ -// | | | MARKING | <-> | YOUNG GC | -// | +----------| | | (RSet Parses Region) | -// | +-----------------+ +----------------------+ -// | | -// | | Has Candidates -// | v -// | +-----------------+ -// | | WAITING | -// +------------- | | +// | | +---> | | | (RSet Uses Bitmap) | +// | | | +-----------------+ +--------------------+ +// | | | | +// | | | | Reset Bitmap +// | | | v +// | | | +-----------------+ +// | | | | BOOTSTRAP | +// | | | | | +// | | | +-----------------+ +// | | | | +// | | | | Continue Marking +// | | | v +// | | | +-----------------+ +----------------------+ +// | | | | MARKING | <-> | YOUNG GC | +// | +----|-----| | | (RSet Parses Region) | +// | | +-----------------+ +----------------------+ +// | | | +// | | | Has Candidates +// | | v +// | | +-----------------+ +// | | | WAITING FOR | +// +--------|---- | EVACUATIONS | +// | +-----------------+ +// | | +// | | All Candidates are Pinned +// | v +// | +-----------------+ +// | | WAITING FOR | +// +-----| FILLING | // +-----------------+ // bool ShenandoahOldGeneration::validate_transition(State new_state) { ShenandoahHeap* heap = ShenandoahHeap::heap(); switch (new_state) { case IDLE: + assert(_state == MARKING || _state == WAITING_FOR_EVAC, "Must come from marking or evacuating."); assert(!heap->is_concurrent_old_mark_in_progress(), "Cannot become idle during old mark."); assert(_old_heuristics->unprocessed_old_collection_candidates() == 0, "Cannot become idle with collection candidates"); assert(!heap->is_prepare_for_old_mark_in_progress(), "Cannot become idle while making old generation parseable."); assert(heap->young_generation()->old_gen_task_queues() == nullptr, "Cannot become idle when setup for bootstrapping."); return true; case FILLING: - assert(_state == IDLE, "Cannot begin filling without first being idle."); + assert(_state == IDLE || _state == WAITING_FOR_FILL, "Cannot begin filling without first completing evacuations."); assert(heap->is_prepare_for_old_mark_in_progress(), "Should be preparing for old mark now."); return true; case BOOTSTRAPPING: assert(_state == FILLING, "Cannot reset bitmap without making old regions parseable."); - // assert(heap->young_generation()->old_gen_task_queues() != nullptr, "Cannot bootstrap without old mark queues."); + assert(_old_heuristics->unprocessed_old_collection_candidates() == 0, "Cannot bootstrap with mixed collection candidates"); assert(!heap->is_prepare_for_old_mark_in_progress(), "Cannot still be making old regions parseable."); return true; case MARKING: @@ -405,10 +418,14 @@ bool ShenandoahOldGeneration::validate_transition(State new_state) { assert(heap->young_generation()->old_gen_task_queues() != nullptr, "Young generation needs old mark queues."); assert(heap->is_concurrent_old_mark_in_progress(), "Should be marking old now."); return true; - case WAITING: + case WAITING_FOR_EVAC: assert(_state == MARKING, "Cannot have old collection candidates without first marking."); assert(_old_heuristics->unprocessed_old_collection_candidates() > 0, "Must have collection candidates here."); return true; + case WAITING_FOR_FILL: + assert(_state == MARKING || _state == WAITING_FOR_EVAC, "Cannot begin filling without first marking or evacuating."); + assert(_old_heuristics->unprocessed_old_collection_candidates() > 0, "Cannot wait for fill without something to fill."); + return true; default: ShouldNotReachHere(); return false; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp index 618a664ce4672..bb6adcfcf768d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp @@ -80,7 +80,7 @@ class ShenandoahOldGeneration : public ShenandoahGeneration { virtual void record_success_concurrent(bool abbreviated) override; enum State { - IDLE, FILLING, BOOTSTRAPPING, MARKING, WAITING + IDLE, FILLING, BOOTSTRAPPING, MARKING, WAITING_FOR_EVAC, WAITING_FOR_FILL }; static const char* state_name(State state); @@ -95,6 +95,10 @@ class ShenandoahOldGeneration : public ShenandoahGeneration { return _state; } + bool can_start_gc() { + return _state == IDLE || _state == WAITING_FOR_FILL; + } + private: bool entry_coalesce_and_fill(); bool coalesce_and_fill(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp index cb740f8e16d9b..afe110cf24d0c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp @@ -451,6 +451,15 @@ product(bool, ShenandoahAllocFailureALot, false, DIAGNOSTIC, \ "Testing: make lots of artificial allocation failures.") \ \ + product(uintx, ShenandoahCoalesceChance, 0, DIAGNOSTIC, \ + "Testing: Abandon remaining mixed collections with this " \ + "likelihood. Following each mixed collection, abandon all " \ + "remaining mixed collection candidate regions with likelihood " \ + "ShenandoahCoalesceChance. Abandoning a mixed collection will " \ + "cause the old regions to be made parseable, rather than being " \ + "evacuated.") \ + range(0, 100) \ + \ product(intx, ShenandoahMarkScanPrefetch, 32, EXPERIMENTAL, \ "How many objects to prefetch ahead when traversing mark bitmaps."\ "Set to 0 to disable prefetching.") \ diff --git a/test/hotspot/gtest/gc/shenandoah/test_shenandoahOldHeuristic.cpp b/test/hotspot/gtest/gc/shenandoah/test_shenandoahOldHeuristic.cpp index 5307ed9584e0b..d5470bae1146c 100644 --- a/test/hotspot/gtest/gc/shenandoah/test_shenandoahOldHeuristic.cpp +++ b/test/hotspot/gtest/gc/shenandoah/test_shenandoahOldHeuristic.cpp @@ -67,6 +67,10 @@ class ShenandoahOldHeuristicTest : public ::testing::Test { _collection_set->clear(); } + ShenandoahOldGeneration::State old_generation_state() { + return _heap->old_generation()->state(); + } + size_t make_garbage(size_t region_idx, size_t garbage_bytes) { ShenandoahHeapLocker locker(_heap->lock()); ShenandoahHeapRegion* region = _heap->get_region(region_idx); @@ -311,4 +315,24 @@ TEST_VM_F(ShenandoahOldHeuristicTest, unpinned_region_is_middle) { EXPECT_EQ(_heuristics->unprocessed_old_collection_candidates(), 0UL); } +TEST_VM_F(ShenandoahOldHeuristicTest, all_candidates_are_pinned) { + SKIP_IF_NOT_SHENANDOAH(); + + size_t g1 = make_garbage_above_threshold(0); + size_t g2 = make_garbage_above_threshold(1); + size_t g3 = make_garbage_above_threshold(2); + + make_pinned(0); + make_pinned(1); + make_pinned(2); + _heuristics->prepare_for_old_collections(); + _heuristics->prime_collection_set(_collection_set); + + // In the case when all candidates are pinned, we want to abandon + // this set of mixed collection candidates so that another old collection + // can run. This is meant to defend against "bad" JNI code that permanently + // leaves an old region in the pinned state. + EXPECT_EQ(_collection_set->count(), 0UL); + EXPECT_EQ(old_generation_state(), ShenandoahOldGeneration::WAITING_FOR_FILL); +} #undef SKIP_IF_NOT_SHENANDOAH From d0e1f5195315019467419dfd377d440026bbb87e Mon Sep 17 00:00:00 2001 From: "Y. Srinivas Ramakrishna" Date: Fri, 31 Mar 2023 20:55:02 +0000 Subject: [PATCH 203/254] 8305334: GenShen: reference processing needs a card-marking barrier Global collections can create new cross-generational pointers during j.l.r.Reference processing which should be added to the card marking remembered set. The issue was found with dacapo during heap verification and happens somewhat rarely. I added the requisite barrier and provided a comment describing the sole situation in reference processing that should need the barrier. Assertions check this condition, but the card is also redundantly dirtied for young collections too where it's strictly not needed. I have tested with and without heap verification using product and fastdebug builds using dacapo and specjbb, and through our internal tests pipelines. I have not run any specific performance comparisons to assess the impact of the checks for applications that traffic heavily in j.l.r.Refs, but I do not expect much impact. Reviewed-by: kdnilsen, wkemper --- .../shenandoahReferenceProcessor.cpp | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp b/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp index b06a714547509..b015ddfaa7ac7 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp @@ -27,6 +27,7 @@ #include "classfile/javaClasses.hpp" #include "gc/shared/workerThread.hpp" #include "gc/shenandoah/mode/shenandoahGenerationalMode.hpp" +#include "gc/shenandoah/shenandoahGeneration.hpp" #include "gc/shenandoah/shenandoahOopClosures.inline.hpp" #include "gc/shenandoah/shenandoahReferenceProcessor.hpp" #include "gc/shenandoah/shenandoahThreadLocalData.hpp" @@ -58,17 +59,44 @@ static const char* reference_type_name(ReferenceType type) { } } +template +static void card_mark_barrier(T* field, oop value) { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + assert(heap->is_in_or_null(value), "Should be in heap"); + if (heap->mode()->is_generational() && heap->is_in_old(field) && heap->is_in_young(value)) { + // We expect this to really be needed only during global collections. Young collections + // discover j.l.r.Refs in the old generation during scanning of dirty cards + // and these point to (as yet unmarked) referents in the young generation (see + // ShenandoahReferenceProcessor::should_discover). Those cards will continue to + // remain dirty on account of this cross-generational pointer to the referent. + // Similarly, old collections will never discover j.l.r.Refs in the young generation. + // It is only global collections that discover in both generations. Here we can + // end up with a j.l.R in the old generation on the discovered list that + // is not already on a dirty card, but which may here end up with a successor in + // the discovered list that is in the young generation. This is the singular case + // where the card needs to be dirtied here. We, however, skip the extra global'ness check + // and always mark the card (redundantly during young collections). + // The asserts below check the expected invariants based on the description above. + assert(!heap->active_generation()->is_old(), "Expecting only young or global"); + assert(heap->card_scan()->is_card_dirty(reinterpret_cast(field)) + || heap->active_generation()->is_global(), "Expecting already dirty if young"); + heap->card_scan()->mark_card_as_dirty(reinterpret_cast(field)); + } +} + template static void set_oop_field(T* field, oop value); template <> void set_oop_field(oop* field, oop value) { *field = value; + card_mark_barrier(field, value); } template <> void set_oop_field(narrowOop* field, oop value) { *field = CompressedOops::encode(value); + card_mark_barrier(field, value); } static oop lrb(oop obj) { From 1143dbebde4fd347df0ca7099e325f3abf2a76fd Mon Sep 17 00:00:00 2001 From: William Kemper Date: Mon, 3 Apr 2023 20:39:47 +0000 Subject: [PATCH 204/254] Allow transition from any old generation state to idle Reviewed-by: kdnilsen --- .../share/gc/shenandoah/shenandoahOldGeneration.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp index ee54da2f4e595..cf57a84b17b3a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp @@ -398,32 +398,32 @@ bool ShenandoahOldGeneration::validate_transition(State new_state) { ShenandoahHeap* heap = ShenandoahHeap::heap(); switch (new_state) { case IDLE: - assert(_state == MARKING || _state == WAITING_FOR_EVAC, "Must come from marking or evacuating."); + // GC cancellation can send us back to IDLE from any state. assert(!heap->is_concurrent_old_mark_in_progress(), "Cannot become idle during old mark."); assert(_old_heuristics->unprocessed_old_collection_candidates() == 0, "Cannot become idle with collection candidates"); assert(!heap->is_prepare_for_old_mark_in_progress(), "Cannot become idle while making old generation parseable."); assert(heap->young_generation()->old_gen_task_queues() == nullptr, "Cannot become idle when setup for bootstrapping."); return true; case FILLING: - assert(_state == IDLE || _state == WAITING_FOR_FILL, "Cannot begin filling without first completing evacuations."); + assert(_state == IDLE || _state == WAITING_FOR_FILL, "Cannot begin filling without first completing evacuations, state is '%s'", state_name(_state)); assert(heap->is_prepare_for_old_mark_in_progress(), "Should be preparing for old mark now."); return true; case BOOTSTRAPPING: - assert(_state == FILLING, "Cannot reset bitmap without making old regions parseable."); + assert(_state == FILLING, "Cannot reset bitmap without making old regions parseable, state is '%s'", state_name(_state)); assert(_old_heuristics->unprocessed_old_collection_candidates() == 0, "Cannot bootstrap with mixed collection candidates"); assert(!heap->is_prepare_for_old_mark_in_progress(), "Cannot still be making old regions parseable."); return true; case MARKING: - assert(_state == BOOTSTRAPPING, "Must have finished bootstrapping before marking."); + assert(_state == BOOTSTRAPPING, "Must have finished bootstrapping before marking, state is '%s'", state_name(_state)); assert(heap->young_generation()->old_gen_task_queues() != nullptr, "Young generation needs old mark queues."); assert(heap->is_concurrent_old_mark_in_progress(), "Should be marking old now."); return true; case WAITING_FOR_EVAC: - assert(_state == MARKING, "Cannot have old collection candidates without first marking."); + assert(_state == MARKING, "Cannot have old collection candidates without first marking, state is '%s'", state_name(_state)); assert(_old_heuristics->unprocessed_old_collection_candidates() > 0, "Must have collection candidates here."); return true; case WAITING_FOR_FILL: - assert(_state == MARKING || _state == WAITING_FOR_EVAC, "Cannot begin filling without first marking or evacuating."); + assert(_state == MARKING || _state == WAITING_FOR_EVAC, "Cannot begin filling without first marking or evacuating, state is '%s'", state_name(_state)); assert(_old_heuristics->unprocessed_old_collection_candidates() > 0, "Cannot wait for fill without something to fill."); return true; default: From fc20c90a0d3fe098305f79ba3e8f02a7f08614c1 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Tue, 4 Apr 2023 16:57:48 +0000 Subject: [PATCH 205/254] Disable redundant generation usage after fullgc Reviewed-by: wkemper --- .../share/gc/shenandoah/shenandoahFullGC.cpp | 114 ++++++------------ .../gc/shenandoah/shenandoahGeneration.cpp | 8 ++ 2 files changed, 46 insertions(+), 76 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index 23fdbf7407854..899a6eab5f75d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -849,7 +849,6 @@ class ShenandoahEnsureHeapActiveClosure: public ShenandoahHeapRegionClosure { public: ShenandoahEnsureHeapActiveClosure() : _heap(ShenandoahHeap::heap()) {} void heap_region_do(ShenandoahHeapRegion* r) { - bool is_generational = _heap->mode()->is_generational(); if (r->is_trash()) { r->recycle(); } @@ -1071,11 +1070,6 @@ void ShenandoahFullGC::phase2_calculate_target_addresses(ShenandoahHeapRegionSet heap->heap_region_iterate(&ecl); } - if (heap->mode()->is_generational()) { - heap->young_generation()->clear_used(); - heap->old_generation()->clear_used(); - } - // Compute the new addresses for regular objects { ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_calculate_addresses_regular); @@ -1266,19 +1260,41 @@ class ShenandoahCompactObjectsTask : public WorkerTask { } }; +static void account_for_region(ShenandoahHeapRegion* r, size_t ®ion_count, size_t ®ion_usage, size_t &humongous_waste) { + region_count++; + region_usage += r->used(); + if (r->is_humongous_start()) { + // For each humongous object, we take this path once regardless of how many regions it spans. + HeapWord* obj_addr = r->bottom(); + oop obj = cast_to_oop(obj_addr); + size_t word_size = obj->size(); + size_t region_size_words = ShenandoahHeapRegion::region_size_words(); + size_t overreach = word_size % region_size_words; + if (overreach != 0) { + humongous_waste += (region_size_words - overreach) * HeapWordSize; + } + // else, this humongous object aligns exactly on region size, so no waste. + } +} + class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure { private: ShenandoahHeap* const _heap; size_t _live; + bool _is_generational; + size_t _young_regions, _young_usage, _young_humongous_waste; + size_t _old_regions, _old_usage, _old_humongous_waste; public: - ShenandoahPostCompactClosure() : _heap(ShenandoahHeap::heap()), _live(0) { + ShenandoahPostCompactClosure() : _heap(ShenandoahHeap::heap()), _live(0), _is_generational(_heap->mode()->is_generational()), + _young_regions(0), _young_usage(0), _young_humongous_waste(0), + _old_regions(0), _old_usage(0), _old_humongous_waste(0) + { _heap->free_set()->clear(); } void heap_region_do(ShenandoahHeapRegion* r) { assert (!r->is_cset(), "cset regions should have been demoted already"); - bool is_generational = _heap->mode()->is_generational(); // Need to reset the complete-top-at-mark-start pointer here because // the complete marking bitmap is no longer valid. This ensures @@ -1291,9 +1307,10 @@ class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure { size_t live = r->used(); + // Make empty regions that have been allocated into regular if (r->is_empty() && live > 0) { - if (!is_generational) { + if (!_is_generational) { r->make_young_maybe(); } // else, generational mode compaction has already established affiliation. @@ -1309,14 +1326,11 @@ class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure { if (r->is_trash()) { live = 0; r->recycle(); - } - - // Update final usage for generations - if (is_generational && live != 0) { - if (r->is_young()) { - _heap->young_generation()->increase_used(live); - } else if (r->is_old()) { - _heap->old_generation()->increase_used(live); + } else if (_is_generational) { + if (r->is_old()) { + account_for_region(r, _old_regions, _old_usage, _old_humongous_waste); + } else if (r->is_young()) { + account_for_region(r, _young_regions, _young_usage, _young_humongous_waste); } } @@ -1328,6 +1342,12 @@ class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure { size_t get_live() { return _live; } + + void update_generation_usage() { + assert(_is_generational, "Only update generation usage if generational"); + _heap->old_generation()->establish_usage(_old_regions, _old_usage, _old_humongous_waste); + _heap->young_generation()->establish_usage(_young_regions, _young_usage, _young_humongous_waste); + } }; void ShenandoahFullGC::compact_humongous_objects() { @@ -1450,62 +1470,9 @@ void ShenandoahFullGC::phase4_compact_objects(ShenandoahHeapRegionSet** worker_s } } -static void account_for_region(ShenandoahHeapRegion* r, size_t ®ion_count, size_t ®ion_usage, size_t &humongous_waste) { - region_count++; - region_usage += r->used(); - if (r->is_humongous_start()) { - // For each humongous object, we take this path once regardless of how many regions it spans. - HeapWord* obj_addr = r->bottom(); - oop obj = cast_to_oop(obj_addr); - size_t word_size = obj->size(); - size_t region_size_words = ShenandoahHeapRegion::region_size_words(); - size_t overreach = word_size % region_size_words; - if (overreach != 0) { - humongous_waste += (region_size_words - overreach) * HeapWordSize; - } - // else, this humongous object aligns exactly on region size, so no waste. - } -} - void ShenandoahFullGC::phase5_epilog() { GCTraceTime(Info, gc, phases) time("Phase 5: Full GC epilog", _gc_timer); ShenandoahHeap* heap = ShenandoahHeap::heap(); - size_t num_regions = heap->num_regions(); - size_t young_usage = 0; - size_t young_regions = 0; - size_t young_humongous_waste = 0; - size_t old_usage = 0; - size_t old_regions = 0; - size_t old_humongous_waste = 0; - ShenandoahHeapRegion* r; - - if (heap->mode()->is_generational()) { - // TODO: We may be able remove code that recomputes generation usage after we fix the incremental updates to generation - // usage that are scattered throughout the existing Full GC implementation. There's an error in there somewhere that - // has not yet been figured out. Or maybe it is easier to just not try to do the generation accounting on the fly, keep - // this code, and remove all of the other attempts to increase/decrease affiliated regions, used, and humongous_waste. - { - ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_recompute_generation_usage); - for (size_t i = 0; i < num_regions; i++) { - switch (heap->region_affiliation(i)) { - case ShenandoahRegionAffiliation::FREE: - break; - case ShenandoahRegionAffiliation::YOUNG_GENERATION: - r = heap->get_region(i); - account_for_region(r, young_regions, young_usage, young_humongous_waste); - break; - case ShenandoahRegionAffiliation::OLD_GENERATION: - r = heap->get_region(i); - account_for_region(r, old_regions, old_usage, old_humongous_waste); - break; - default: - assert(false, "Should not reach"); - } - } - heap->old_generation()->establish_usage(old_regions, old_usage, old_humongous_waste); - heap->young_generation()->establish_usage(young_regions, young_usage, young_humongous_waste); - } - } // Reset complete bitmap. We're about to reset the complete-top-at-mark-start pointer // and must ensure the bitmap is in sync. @@ -1518,16 +1485,11 @@ void ShenandoahFullGC::phase5_epilog() { // Bring regions in proper states after the collection, and set heap properties. { ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_rebuild); - - if (heap->mode()->is_generational()) { - heap->young_generation()->clear_used(); - heap->old_generation()->clear_used(); - } - ShenandoahPostCompactClosure post_compact; heap->heap_region_iterate(&post_compact); heap->set_used(post_compact.get_live()); if (heap->mode()->is_generational()) { + post_compact.update_generation_usage(); log_info(gc)("FullGC done: GLOBAL usage: " SIZE_FORMAT ", young usage: " SIZE_FORMAT ", old usage: " SIZE_FORMAT, post_compact.get_live(), heap->young_generation()->used(), heap->old_generation()->used()); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index cef6aed6e1ae1..147a385d9b41f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -933,11 +933,19 @@ void ShenandoahGeneration::scan_remembered_set(bool is_concurrent) { } size_t ShenandoahGeneration::increment_affiliated_region_count() { + shenandoah_assert_heaplocked_or_fullgc_safepoint(); + // During full gc, multiple GC worker threads may change region affiliations without a lock. No lock is enforced + // on read and write of _affiliated_region_count. At the end of full gc, a single thread overwrites the count with + // a coherent value. _affiliated_region_count++; return _affiliated_region_count; } size_t ShenandoahGeneration::decrement_affiliated_region_count() { + shenandoah_assert_heaplocked_or_fullgc_safepoint(); + // During full gc, multiple GC worker threads may change region affiliations without a lock. No lock is enforced + // on read and write of _affiliated_region_count. At the end of full gc, a single thread overwrites the count with + // a coherent value. _affiliated_region_count--; return _affiliated_region_count; } From ab9a39713fafc2257cc34531fe638e82b7221067 Mon Sep 17 00:00:00 2001 From: Aleksey Shipilev Date: Tue, 4 Apr 2023 19:09:16 +0000 Subject: [PATCH 206/254] Fix build error on MacOS / Clang 14.0.0 Reviewed-by: wkemper --- .../share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp index a12567fd3c739..eed460c33aba7 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp @@ -48,7 +48,7 @@ const char* generation_name(GenerationMode mode); class ShenandoahGenerationalMode : public ShenandoahMode { public: virtual void initialize_flags() const; - virtual ShenandoahHeuristics* initialize_heuristics(ShenandoahGeneration* generation) const override; + virtual ShenandoahHeuristics* initialize_heuristics(ShenandoahGeneration* generation) const; virtual const char* name() { return "Generational"; } virtual bool is_diagnostic() { return false; } virtual bool is_experimental() { return false; } From 72b67addbb2f583069d0fe14e3702b16c46024c0 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Tue, 4 Apr 2023 19:53:33 +0000 Subject: [PATCH 207/254] Remove assertions constraining card marking for reference processing Reviewed-by: kdnilsen --- .../shenandoahReferenceProcessor.cpp | 23 ++++++------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp b/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp index b015ddfaa7ac7..10840f7373000 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp @@ -64,22 +64,13 @@ static void card_mark_barrier(T* field, oop value) { ShenandoahHeap* heap = ShenandoahHeap::heap(); assert(heap->is_in_or_null(value), "Should be in heap"); if (heap->mode()->is_generational() && heap->is_in_old(field) && heap->is_in_young(value)) { - // We expect this to really be needed only during global collections. Young collections - // discover j.l.r.Refs in the old generation during scanning of dirty cards - // and these point to (as yet unmarked) referents in the young generation (see - // ShenandoahReferenceProcessor::should_discover). Those cards will continue to - // remain dirty on account of this cross-generational pointer to the referent. - // Similarly, old collections will never discover j.l.r.Refs in the young generation. - // It is only global collections that discover in both generations. Here we can - // end up with a j.l.R in the old generation on the discovered list that - // is not already on a dirty card, but which may here end up with a successor in - // the discovered list that is in the young generation. This is the singular case - // where the card needs to be dirtied here. We, however, skip the extra global'ness check - // and always mark the card (redundantly during young collections). - // The asserts below check the expected invariants based on the description above. - assert(!heap->active_generation()->is_old(), "Expecting only young or global"); - assert(heap->card_scan()->is_card_dirty(reinterpret_cast(field)) - || heap->active_generation()->is_global(), "Expecting already dirty if young"); + // For Shenandoah, each generation collects all the _referents_ that belong to the + // collected generation. We can end up with discovered lists that contain a mixture + // of old and young _references_. These references are linked together through the + // discovered field in java.lang.Reference. In some cases, creating or editing this + // list may result in the creation of _new_ old-to-young pointers which must dirty + // the corresponding card. Failing to do this may cause heap verification errors and + // lead to incorrect GC behavior. heap->card_scan()->mark_card_as_dirty(reinterpret_cast(field)); } } From 3a03d7dd921c0de13e8ace19bf11722055de9748 Mon Sep 17 00:00:00 2001 From: Aleksey Shipilev Date: Wed, 5 Apr 2023 07:53:53 +0000 Subject: [PATCH 208/254] Fix Zero builds Reviewed-by: kdnilsen --- .../gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp index c61e53299f2fc..708c16f7d4bf1 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp @@ -27,6 +27,7 @@ #include "gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp" #include "gc/shenandoah/shenandoahCollectionSet.hpp" #include "gc/shenandoah/shenandoahFreeSet.hpp" +#include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahGeneration.hpp" #include "gc/shenandoah/shenandoahYoungGeneration.hpp" #include "logging/log.hpp" From d6df935b0b280e7cde9a97a49f2578615a3812b1 Mon Sep 17 00:00:00 2001 From: Aleksey Shipilev Date: Thu, 6 Apr 2023 16:53:40 +0000 Subject: [PATCH 209/254] Update copyright lines Reviewed-by: wkemper --- .../c1/shenandoahBarrierSetC1_aarch64.cpp | 1 + .../shenandoahBarrierSetAssembler_aarch64.cpp | 1 + .../shenandoahBarrierSetAssembler_aarch64.hpp | 1 + .../c1/shenandoahBarrierSetC1_ppc.cpp | 1 + .../shenandoahBarrierSetAssembler_ppc.cpp | 1 + .../shenandoahBarrierSetAssembler_ppc.hpp | 1 + .../c1/shenandoahBarrierSetC1_riscv.cpp | 1 + .../shenandoahBarrierSetAssembler_riscv.cpp | 1 + .../c1/shenandoahBarrierSetC1_x86.cpp | 1 + .../shenandoahBarrierSetAssembler_x86.cpp | 1 + .../shenandoahBarrierSetAssembler_x86.hpp | 1 + src/hotspot/share/gc/shared/ageTable.cpp | 2 +- src/hotspot/share/gc/shared/ageTable.hpp | 2 +- .../share/gc/shared/gcConfiguration.cpp | 2 +- src/hotspot/share/gc/shared/plab.hpp | 1 + .../shenandoah/c1/shenandoahBarrierSetC1.cpp | 1 + .../shenandoah/c1/shenandoahBarrierSetC1.hpp | 1 + .../shenandoah/c2/shenandoahBarrierSetC2.cpp | 1 + .../shenandoah/c2/shenandoahBarrierSetC2.hpp | 1 + .../gc/shenandoah/c2/shenandoahSupport.cpp | 1 + .../shenandoahAdaptiveHeuristics.cpp | 1 + .../shenandoahAdaptiveHeuristics.hpp | 1 + .../shenandoahAggressiveHeuristics.cpp | 1 + .../shenandoahAggressiveHeuristics.hpp | 1 + .../shenandoahCompactHeuristics.cpp | 1 + .../shenandoahCompactHeuristics.hpp | 1 + .../heuristics/shenandoahHeuristics.cpp | 3 ++- .../heuristics/shenandoahHeuristics.hpp | 1 + .../heuristics/shenandoahOldHeuristics.cpp | 2 +- .../heuristics/shenandoahOldHeuristics.hpp | 2 +- .../shenandoahPassiveHeuristics.cpp | 1 + .../shenandoahPassiveHeuristics.hpp | 1 + .../heuristics/shenandoahStaticHeuristics.cpp | 1 + .../heuristics/shenandoahStaticHeuristics.hpp | 1 + .../mode/shenandoahGenerationalMode.cpp | 2 +- .../mode/shenandoahGenerationalMode.hpp | 2 +- .../gc/shenandoah/mode/shenandoahMode.cpp | 1 + .../gc/shenandoah/mode/shenandoahMode.hpp | 3 ++- .../shenandoah/mode/shenandoahPassiveMode.hpp | 3 ++- .../gc/shenandoah/mode/shenandoahSATBMode.cpp | 1 + .../gc/shenandoah/shenandoahAllocRequest.hpp | 3 ++- .../gc/shenandoah/shenandoahArguments.cpp | 1 + .../share/gc/shenandoah/shenandoahAsserts.hpp | 1 + .../gc/shenandoah/shenandoahBarrierSet.cpp | 1 + .../gc/shenandoah/shenandoahBarrierSet.hpp | 1 + .../shenandoahBarrierSet.inline.hpp | 1 + .../shenandoahBarrierSetClone.inline.hpp | 1 + .../gc/shenandoah/shenandoahCardStats.cpp | 3 +-- .../gc/shenandoah/shenandoahCardStats.hpp | 3 +-- .../gc/shenandoah/shenandoahCardTable.cpp | 2 +- .../gc/shenandoah/shenandoahCardTable.hpp | 2 +- .../shenandoah/shenandoahClosures.inline.hpp | 1 + .../gc/shenandoah/shenandoahCollectionSet.cpp | 1 + .../gc/shenandoah/shenandoahCollectionSet.hpp | 1 + .../shenandoahCollectionSet.inline.hpp | 1 + .../shenandoah/shenandoahCollectorPolicy.cpp | 1 + .../shenandoah/shenandoahCollectorPolicy.hpp | 1 + .../gc/shenandoah/shenandoahConcurrentGC.cpp | 1 + .../gc/shenandoah/shenandoahConcurrentGC.hpp | 1 + .../shenandoah/shenandoahConcurrentMark.cpp | 1 + .../shenandoah/shenandoahConcurrentMark.hpp | 1 + .../gc/shenandoah/shenandoahControlThread.cpp | 1 + .../gc/shenandoah/shenandoahControlThread.hpp | 1 + .../gc/shenandoah/shenandoahDegeneratedGC.cpp | 1 + .../gc/shenandoah/shenandoahDegeneratedGC.hpp | 1 + .../gc/shenandoah/shenandoahEvacTracker.cpp | 2 +- .../gc/shenandoah/shenandoahEvacTracker.hpp | 2 +- .../share/gc/shenandoah/shenandoahFreeSet.cpp | 1 + .../share/gc/shenandoah/shenandoahFreeSet.hpp | 1 + .../share/gc/shenandoah/shenandoahFullGC.cpp | 1 + .../share/gc/shenandoah/shenandoahFullGC.hpp | 1 + .../share/gc/shenandoah/shenandoahGC.cpp | 1 + .../share/gc/shenandoah/shenandoahGC.hpp | 1 + .../gc/shenandoah/shenandoahGeneration.cpp | 2 +- .../gc/shenandoah/shenandoahGeneration.hpp | 2 +- .../shenandoah/shenandoahGlobalGeneration.cpp | 2 +- .../shenandoah/shenandoahGlobalGeneration.hpp | 2 +- .../share/gc/shenandoah/shenandoahHeap.cpp | 1 + .../share/gc/shenandoah/shenandoahHeap.hpp | 1 + .../gc/shenandoah/shenandoahHeap.inline.hpp | 1 + .../gc/shenandoah/shenandoahHeapRegion.cpp | 1 + .../gc/shenandoah/shenandoahHeapRegion.hpp | 3 ++- .../shenandoahHeapRegion.inline.hpp | 1 + .../shenandoahHeapRegionCounters.cpp | 1 + .../shenandoahHeapRegionCounters.hpp | 1 + .../gc/shenandoah/shenandoahInitLogger.cpp | 1 + .../share/gc/shenandoah/shenandoahMark.cpp | 1 + .../share/gc/shenandoah/shenandoahMark.hpp | 1 + .../gc/shenandoah/shenandoahMark.inline.hpp | 1 + .../gc/shenandoah/shenandoahMarkBitMap.cpp | 1 + .../gc/shenandoah/shenandoahMarkBitMap.hpp | 1 + .../gc/shenandoah/shenandoahMarkClosures.cpp | 1 + .../gc/shenandoah/shenandoahMarkClosures.hpp | 2 +- .../shenandoah/shenandoahMarkingContext.cpp | 1 + .../shenandoah/shenandoahMarkingContext.hpp | 1 + .../shenandoahMarkingContext.inline.hpp | 1 + .../gc/shenandoah/shenandoahMemoryPool.cpp | 1 + .../gc/shenandoah/shenandoahMemoryPool.hpp | 1 + .../gc/shenandoah/shenandoahMmuTracker.cpp | 2 +- .../gc/shenandoah/shenandoahMmuTracker.hpp | 2 +- .../share/gc/shenandoah/shenandoahNMethod.cpp | 1 + .../gc/shenandoah/shenandoahNumberSeq.cpp | 1 + .../gc/shenandoah/shenandoahNumberSeq.hpp | 1 + .../share/gc/shenandoah/shenandoahOldGC.cpp | 2 +- .../share/gc/shenandoah/shenandoahOldGC.hpp | 2 +- .../gc/shenandoah/shenandoahOldGeneration.cpp | 2 +- .../gc/shenandoah/shenandoahOldGeneration.hpp | 2 +- .../gc/shenandoah/shenandoahOopClosures.hpp | 1 + .../shenandoahOopClosures.inline.hpp | 1 + .../gc/shenandoah/shenandoahPhaseTimings.cpp | 1 + .../gc/shenandoah/shenandoahPhaseTimings.hpp | 1 + .../shenandoahReferenceProcessor.cpp | 1 + .../shenandoah/shenandoahRegulatorThread.cpp | 2 +- .../shenandoah/shenandoahRegulatorThread.hpp | 2 +- .../gc/shenandoah/shenandoahRootVerifier.cpp | 1 + .../gc/shenandoah/shenandoahRootVerifier.hpp | 1 + .../share/gc/shenandoah/shenandoahSTWMark.cpp | 1 + .../share/gc/shenandoah/shenandoahSTWMark.hpp | 1 + .../shenandoah/shenandoahScanRemembered.cpp | 3 +-- .../shenandoah/shenandoahScanRemembered.hpp | 3 +-- .../shenandoahScanRemembered.inline.hpp | 3 +-- .../shenandoah/shenandoahStackWatermark.cpp | 1 + .../shenandoah/shenandoahThreadLocalData.hpp | 1 + .../share/gc/shenandoah/shenandoahUnload.cpp | 1 + .../share/gc/shenandoah/shenandoahUtils.cpp | 1 + .../share/gc/shenandoah/shenandoahUtils.hpp | 1 + .../gc/shenandoah/shenandoahVMOperations.cpp | 1 + .../gc/shenandoah/shenandoahVMOperations.hpp | 1 + .../gc/shenandoah/shenandoahVerifier.cpp | 1 + .../gc/shenandoah/shenandoahVerifier.hpp | 1 + .../gc/shenandoah/shenandoahWorkerPolicy.cpp | 1 + .../gc/shenandoah/shenandoahWorkerPolicy.hpp | 1 + .../shenandoah/shenandoahYoungGeneration.cpp | 2 +- .../shenandoah/shenandoahYoungGeneration.hpp | 2 +- .../gc/shenandoah/shenandoah_globals.hpp | 3 ++- src/hotspot/share/utilities/numberSeq.hpp | 2 +- .../shenandoah/test_shenandoahNumberSeq.cpp | 3 ++- .../test_shenandoahOldHeuristic.cpp | 23 +++++++++++++++++++ .../jtreg/gc/TestAllocHumongousFragment.java | 1 + .../gc/shenandoah/TestAllocIntArrays.java | 1 + .../gc/shenandoah/TestAllocObjectArrays.java | 1 + .../jtreg/gc/shenandoah/TestAllocObjects.java | 1 + .../gc/shenandoah/TestArrayCopyCheckCast.java | 1 + .../gc/shenandoah/TestArrayCopyStress.java | 1 + .../TestDynamicSoftMaxHeapSize.java | 1 + .../jtreg/gc/shenandoah/TestElasticTLAB.java | 1 + .../jtreg/gc/shenandoah/TestEvilSyncBug.java | 1 + .../gc/shenandoah/TestGCThreadGroups.java | 1 + .../jtreg/gc/shenandoah/TestHeapUncommit.java | 1 + .../gc/shenandoah/TestHumongousThreshold.java | 1 + .../jtreg/gc/shenandoah/TestJcmdHeapDump.java | 1 + .../shenandoah/TestLargeObjectAlignment.java | 1 + .../jtreg/gc/shenandoah/TestLotsOfCycles.java | 1 + .../gc/shenandoah/TestObjItrWithHeapDump.java | 1 + .../jtreg/gc/shenandoah/TestPeriodicGC.java | 1 + .../TestReferenceRefersToShenandoah.java | 2 +- .../TestReferenceShortcutCycle.java | 2 +- .../gc/shenandoah/TestRefprocSanity.java | 1 + .../gc/shenandoah/TestRegionSampling.java | 1 + .../shenandoah/TestRegionSamplingLogging.java | 2 +- .../jtreg/gc/shenandoah/TestResizeTLAB.java | 1 + .../gc/shenandoah/TestRetainObjects.java | 1 + .../shenandoah/TestShenandoahLogRotation.java | 2 +- .../jtreg/gc/shenandoah/TestSieveObjects.java | 1 + .../jtreg/gc/shenandoah/TestSmallHeap.java | 1 + .../jtreg/gc/shenandoah/TestStringDedup.java | 1 + .../gc/shenandoah/TestStringDedupStress.java | 1 + .../shenandoah/TestStringInternCleanup.java | 1 + .../gc/shenandoah/TestVerifyJCStress.java | 1 + .../jtreg/gc/shenandoah/TestVerifyLevels.java | 1 + .../jtreg/gc/shenandoah/TestWithLogLevel.java | 1 + .../gc/shenandoah/TestWrongArrayMember.java | 1 + .../gc/shenandoah/compiler/TestClone.java | 1 + .../shenandoah/compiler/TestReferenceCAS.java | 1 + .../generational/TestCLIModeGenerational.java | 2 +- .../generational/TestConcurrentEvac.java | 2 +- .../generational/TestSimpleGenerational.java | 2 +- .../gc/shenandoah/jni/TestJNICritical.java | 1 + .../gc/shenandoah/jni/TestJNIGlobalRefs.java | 1 + .../gc/shenandoah/jni/TestPinnedGarbage.java | 1 + .../gc/shenandoah/jvmti/TestHeapDump.java | 1 + .../mxbeans/TestChurnNotifications.java | 1 + .../shenandoah/mxbeans/TestMemoryMXBeans.java | 1 + .../shenandoah/mxbeans/TestMemoryPools.java | 1 + .../mxbeans/TestPauseNotifications.java | 1 + .../shenandoah/oom/TestClassLoaderLeak.java | 1 + .../gc/shenandoah/oom/TestThreadFailure.java | 1 + .../gc/shenandoah/options/TestModeUnlock.java | 1 + .../gcbasher/TestGCBasherWithShenandoah.java | 1 + .../gclocker/TestGCLockerWithShenandoah.java | 1 + .../stress/gcold/TestGCOldWithShenandoah.java | 1 + ...ssBigAllocationGCEventsWithShenandoah.java | 3 ++- .../systemgc/TestSystemGCWithShenandoah.java | 1 + 193 files changed, 223 insertions(+), 52 deletions(-) diff --git a/src/hotspot/cpu/aarch64/gc/shenandoah/c1/shenandoahBarrierSetC1_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shenandoah/c1/shenandoahBarrierSetC1_aarch64.cpp index 19057611fa355..ef1e14e910e73 100644 --- a/src/hotspot/cpu/aarch64/gc/shenandoah/c1/shenandoahBarrierSetC1_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/gc/shenandoah/c1/shenandoahBarrierSetC1_aarch64.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp index 527182258e132..36d9097dc0b44 100644 --- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, 2022, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp index 7fffba291f30a..e7b7ba40f381c 100644 --- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/cpu/ppc/gc/shenandoah/c1/shenandoahBarrierSetC1_ppc.cpp b/src/hotspot/cpu/ppc/gc/shenandoah/c1/shenandoahBarrierSetC1_ppc.cpp index 02d219973a953..6f2dd54deff96 100644 --- a/src/hotspot/cpu/ppc/gc/shenandoah/c1/shenandoahBarrierSetC1_ppc.cpp +++ b/src/hotspot/cpu/ppc/gc/shenandoah/c1/shenandoahBarrierSetC1_ppc.cpp @@ -1,6 +1,7 @@ /* * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. * Copyright (c) 2012, 2021 SAP SE. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp b/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp index 3e7ff0030b0d4..02f5fd40d5659 100644 --- a/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp +++ b/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp @@ -1,6 +1,7 @@ /* * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. * Copyright (c) 2012, 2021 SAP SE. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.hpp b/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.hpp index 7ab4ebe16ee7a..2c08f13fab823 100644 --- a/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.hpp +++ b/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.hpp @@ -1,6 +1,7 @@ /* * Copyright (c) 2018, 2022, Red Hat, Inc. All rights reserved. * Copyright (c) 2012, 2022 SAP SE. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/cpu/riscv/gc/shenandoah/c1/shenandoahBarrierSetC1_riscv.cpp b/src/hotspot/cpu/riscv/gc/shenandoah/c1/shenandoahBarrierSetC1_riscv.cpp index cd568cc723fe9..5334f3d165ec5 100644 --- a/src/hotspot/cpu/riscv/gc/shenandoah/c1/shenandoahBarrierSetC1_riscv.cpp +++ b/src/hotspot/cpu/riscv/gc/shenandoah/c1/shenandoahBarrierSetC1_riscv.cpp @@ -1,6 +1,7 @@ /* * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved. * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.cpp b/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.cpp index 07aa390c5399e..96bee969f65e8 100644 --- a/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.cpp +++ b/src/hotspot/cpu/riscv/gc/shenandoah/shenandoahBarrierSetAssembler_riscv.cpp @@ -1,6 +1,7 @@ /* * Copyright (c) 2018, 2020, Red Hat, Inc. All rights reserved. * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/cpu/x86/gc/shenandoah/c1/shenandoahBarrierSetC1_x86.cpp b/src/hotspot/cpu/x86/gc/shenandoah/c1/shenandoahBarrierSetC1_x86.cpp index b47fd038d1512..de50bcd3799f4 100644 --- a/src/hotspot/cpu/x86/gc/shenandoah/c1/shenandoahBarrierSetC1_x86.cpp +++ b/src/hotspot/cpu/x86/gc/shenandoah/c1/shenandoahBarrierSetC1_x86.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp index e33a2bbe81452..9b5bccc158d79 100644 --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp index 68909c11ff5fe..77da40a66d531 100644 --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shared/ageTable.cpp b/src/hotspot/share/gc/shared/ageTable.cpp index 50b1031ecad89..2f21051d7b283 100644 --- a/src/hotspot/share/gc/shared/ageTable.cpp +++ b/src/hotspot/share/gc/shared/ageTable.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shared/ageTable.hpp b/src/hotspot/share/gc/shared/ageTable.hpp index 9f0c10ec31203..96250731ed27d 100644 --- a/src/hotspot/share/gc/shared/ageTable.hpp +++ b/src/hotspot/share/gc/shared/ageTable.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shared/gcConfiguration.cpp b/src/hotspot/share/gc/shared/gcConfiguration.cpp index 386568368dee0..c67c6bcc768c1 100644 --- a/src/hotspot/share/gc/shared/gcConfiguration.cpp +++ b/src/hotspot/share/gc/shared/gcConfiguration.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012, 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2012, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shared/plab.hpp b/src/hotspot/share/gc/shared/plab.hpp index a34e420d44746..a7c8c0aeed772 100644 --- a/src/hotspot/share/gc/shared/plab.hpp +++ b/src/hotspot/share/gc/shared/plab.hpp @@ -7,6 +7,7 @@ * published by the Free Software Foundation. * * This code is distributed in the hope that it will be useful, but WITHOUT + * * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * version 2 for more details (a copy is included in the LICENSE file that diff --git a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp index cac0bc4d575a7..068e09e4ee0c3 100644 --- a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp +++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, 2022, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp index 8b82152973b31..20d0cf35c7532 100644 --- a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp +++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp index d4be3fbc38474..f72d292c88556 100644 --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp index 2df2bcd5c0a64..73134f7601e96 100644 --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp index 61b43ae41fe33..1b5e7bb29cb04 100644 --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp @@ -1,6 +1,7 @@ /* * Copyright (c) 2015, 2021, Red Hat, Inc. All rights reserved. * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp index 708c16f7d4bf1..acc8a4820f1b0 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp index d428babd36a70..85b563f146d20 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp index dde3144952052..d2f40589b7abb 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.hpp index 4f11d96377e47..fede6aff10a4d 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp index 70a3db019be00..d96f7ce0a83ab 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.hpp index eb877921a7483..cd61f1a50d54e 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp index e6cf7e19aa66d..019ad0c054d70 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp @@ -1,5 +1,6 @@ /* - * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. + * Copyright (c) 2018, 2020, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp index 31110867c455a..5e55ce13213f0 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp index a4da2f460a63c..883632ca309e4 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, Amazon.com, Inc. or its affiliates. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp index 9eb065ef897b2..f8581c2cd01a5 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, Amazon.com, Inc. or its affiliates. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp index f4df45133a452..1fa6ec034fcbc 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.hpp index a41f2e82cb4fc..701f2362d5737 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp index 255cca792edf4..779677e221354 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.hpp index 35a41aff8cb93..d71d9c35820d4 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp index a34005581385a..a9d07efe884a2 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, 2020, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp index eed460c33aba7..5cf4fdaaa7979 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.cpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.cpp index b7775e9e9f87e..d6be1f7272961 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.cpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2020, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.hpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.hpp index 46e2079d593ea..826e6ddf2cac3 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.hpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahMode.hpp @@ -1,5 +1,6 @@ /* - * Copyright (c) 2019, 2021, Red Hat, Inc. All rights reserved. + * Copyright (c) 2019, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.hpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.hpp index 7b3f6da92bcb2..7a7006fdb3cb8 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.hpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.hpp @@ -1,5 +1,6 @@ /* - * Copyright (c) 2019, 2021, Red Hat, Inc. All rights reserved. + * Copyright (c) 2019, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahSATBMode.cpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahSATBMode.cpp index 275ab6463c55c..5076d92c0ea09 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahSATBMode.cpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahSATBMode.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2019, 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahAllocRequest.hpp b/src/hotspot/share/gc/shenandoah/shenandoahAllocRequest.hpp index e210666348e77..f05057c88df14 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahAllocRequest.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahAllocRequest.hpp @@ -1,5 +1,6 @@ /* - * Copyright (c) 2018, 2020, Red Hat, Inc. All rights reserved. + * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp b/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp index 0d05cb936dea9..2400d82e700c2 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahArguments.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, 2022, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahAsserts.hpp b/src/hotspot/share/gc/shenandoah/shenandoahAsserts.hpp index 778d42a7fdedc..af10169396ee0 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahAsserts.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahAsserts.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp index 92508f1a64722..ea74454411e35 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2013, 2022, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp index 16453e3275e87..f039120a92c31 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp index a4bf8043ab7ac..db17c6bcf4fc6 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2015, 2022, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp index 0ca2c6da53907..515a3815e3f48 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCardStats.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCardStats.cpp index cb06d99edbacf..abd34c2cbf04a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCardStats.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCardStats.cpp @@ -1,6 +1,5 @@ /* - * Copyright (c) 2022, Amazon.com, Inc. or its affiliates. All rights reserved. - * + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCardStats.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCardStats.hpp index abdc6f752049e..548b2e0174561 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCardStats.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCardStats.hpp @@ -1,6 +1,5 @@ /* - * Copyright (c) 2022, Amazon.com, Inc. or its affiliates. All rights reserved. - * + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCardTable.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCardTable.cpp index 8521f98df112a..89714d266ac95 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCardTable.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCardTable.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2021, Amazon.com, Inc. and/or its affiliates. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCardTable.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCardTable.hpp index f5523c055c38d..bd0f5121e19a7 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCardTable.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCardTable.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2021, Amazon.com, Inc. and/or its affiliates. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp index d1c91d5bf0de2..c7e9160332972 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahClosures.inline.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2019, 2020, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp index 7acf8d7ce2112..59cfbf8f6236e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2016, 2023, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp index 163e2f1cca463..5b497ef50519e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2016, 2020, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.inline.hpp index 1c9d6a99d0547..bc55f6671b8e5 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.inline.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2017, 2020, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.cpp index b462b6b1156f0..73e1d5dbf2cf8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp index 1f8265a0e4acf..3ca9965dbda89 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index 2be57a2b4ab0d..18d9869470ec1 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2021, 2022, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp index 6d431c645fd51..2ce7185d3cf51 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp index 25cb4eb678e73..0cea8b47fea0e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp index 4741a05b4c93b..e74e0e6e7f9fc 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index 48b8e4b26856b..d744b418506bc 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -1,6 +1,7 @@ /* * Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved. * Copyright (C) 2022 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp index 6f1974574e5db..b75795692d6bd 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp index 249a75b8b05c9..843e0c8a451b0 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.hpp index 7b0ffe253ac30..9e92e5b2aa07d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.cpp b/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.cpp index 59771292a562d..a49d8a13a8b7e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, Amazon, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.hpp b/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.hpp index 18087d762ca98..9a864cf3a0581 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, Amazon, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index e610b75f7f1bf..aeaf0bb67481c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2016, 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp index da944e7fa44d2..b0f449770cfb3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2016, 2019, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index 899a6eab5f75d..f251860a3ccb5 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.hpp index 6687116b21f78..4c28684c4050b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGC.cpp index b0731b618f782..767465475df99 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGC.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGC.hpp index 4e929363c947a..0caac37c959f8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGC.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index 147a385d9b41f..458482e1e0b06 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2021 Amazon.com, Inc. and/or its affiliates. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index b12af084d112e..b8c2f8bf8be8a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2021 Amazon.com, Inc. and/or its affiliates. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp index 8bda79d480582..3259c91aa5f62 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp index fb5e043110e6b..e6cd04835b2b8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, Amazon.com, Inc. and/or its affiliates. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 7cb293d820dbe..97892451b5c23 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -1,6 +1,7 @@ /* * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2013, 2022, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index 940198ff1afae..b0a5d8ce459d3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -1,6 +1,7 @@ /* * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp index bdbfb6d05d2e3..947ba01fb2fb4 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2015, 2020, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index 9eb4317db70d6..2dc88cdba2fd8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -1,6 +1,7 @@ /* * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2013, 2020, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp index d4e7d66d20290..951c1fc7dbb07 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp @@ -1,5 +1,6 @@ /* - * Copyright (c) 2013, 2020, Red Hat, Inc. All rights reserved. + * Copyright (c) 2013, 2019, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp index 45dc554275d33..6ed0a208c2603 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2015, 2019, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp index b10816d7e98b4..810e3b816f251 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2016, 2020, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.hpp index 1376ff747d13d..5d56b9c3c0928 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2016, 2019, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahInitLogger.cpp b/src/hotspot/share/gc/shenandoah/shenandoahInitLogger.cpp index dd3176520a901..e9268063a563c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahInitLogger.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahInitLogger.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2020, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp index 3dfd7c919539d..ef98f27464927 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2021, 2022, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.hpp index ac496d7c72ac1..d5e8a6f645653 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMark.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2021, 2022, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp index 7828d3725563f..6101aa8c48216 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2015, 2022, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkBitMap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkBitMap.cpp index affaf71be9c49..28eb2a9a5153a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkBitMap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkBitMap.cpp @@ -1,6 +1,7 @@ /* * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2020, Red Hat, Inc. and/or its affiliates. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkBitMap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkBitMap.hpp index b088d31c7922f..e262429edbf64 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkBitMap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkBitMap.hpp @@ -1,6 +1,7 @@ /* * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2020, Red Hat, Inc. and/or its affiliates. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.cpp index ba0dadf6fc148..afa11ec03d706 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.hpp index 304541cb7e39c..1d7ece9389fcf 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkClosures.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.cpp index 6afa6ca0689dd..b7d503205b5c8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.hpp index 2c3b51eaae0f4..3b73384741553 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.inline.hpp index 0433b4b1f5360..2e99dd6bff4fe 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.inline.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMemoryPool.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMemoryPool.cpp index 6821751412034..79d3d7c2e8ad1 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMemoryPool.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMemoryPool.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2013, 2019, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMemoryPool.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMemoryPool.hpp index 29318ad249955..2743e65edd225 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMemoryPool.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMemoryPool.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2013, 2019, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp index 2c724337b4736..568f070b6df31 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, Amazon, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.hpp index f166a74b32998..9d3c230a6cc57 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, Amazon, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp b/src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp index 65d61b6f74495..44025ddd01b9c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2019, 2022, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.cpp b/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.cpp index adeb669893208..5f77bcc44c90c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.hpp b/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.hpp index 6dd4cd6a93fb1..2249d1ac2fa98 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp index 5dd017793b030..b74d837071e0c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, Amazon.com, Inc. or its affiliates. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.hpp index 671ef4f90e70c..e6ca77226d2a5 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, Amazon.com, Inc. or its affiliates. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp index cf57a84b17b3a..c4433d788bba5 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, Amazon.com, Inc. or its affiliates. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp index bb6adcfcf768d..6450f1fd39faf 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, Amazon.com, Inc. or its affiliates. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp index af80f80af65bb..1d7664205e9b9 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2015, 2022, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp index 8a467019ad4a8..5110ea1f6e351 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2015, 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.cpp b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.cpp index d19f8f25451eb..f8faaf62d1e57 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2017, 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp index d3d9f4d2b91fd..48a7ec12d9c92 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2017, 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp b/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp index 10840f7373000..e6beb585cdf00 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp @@ -1,6 +1,7 @@ /* * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2020, 2021, Red Hat, Inc. and/or its affiliates. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.cpp index fa18cdf5d0a34..2cbbe6b534760 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, Amazon.com, Inc. and/or its affiliates. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.hpp b/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.hpp index a07ae7ded3fff..347adbda46ed3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, Amazon.com, Inc. and/or its affiliates. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.cpp b/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.cpp index 07360bcae68e0..9766660138ae8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2019, 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.hpp b/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.hpp index d2715f4a3dfd2..da7ca864dbbf1 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRootVerifier.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2019, 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp index fd4194475570a..246c11b2f572a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2021, 2022, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.hpp b/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.hpp index 265f404f7b8b2..694cca0c005a7 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp index fcd96548cb632..2ba22813c215d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp @@ -1,6 +1,5 @@ /* - * Copyright (c) 2021, Amazon.com, Inc. or its affiliates. All rights reserved. - * + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp index 0f77319cee63a..16b794c2c700e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp @@ -1,6 +1,5 @@ /* - * Copyright (c) 2021, Amazon.com, Inc. or its affiliates. All rights reserved. - * + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp index 68e2105d72162..c4a807fce3cc3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp @@ -1,6 +1,5 @@ /* - * Copyright (c) 2021, Amazon.com, Inc. or its affiliates. All rights reserved. - * + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahStackWatermark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahStackWatermark.cpp index 6628beecb3280..d3b5bd509f595 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahStackWatermark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahStackWatermark.cpp @@ -1,6 +1,7 @@ /* * Copyright (c) 2021, Red Hat, Inc. All rights reserved. * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp b/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp index 6920e63b03be7..191a9725ca0d8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, 2022, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahUnload.cpp b/src/hotspot/share/gc/shenandoah/shenandoahUnload.cpp index 9987ef361ea5a..b8f85436c840d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahUnload.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahUnload.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2019, 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp b/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp index b43d2b9f490ea..93658e6d44e61 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2017, 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp b/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp index 8c2e903675be2..6dac5e46e2a32 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2017, 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp index 07dab323c1a65..8f8a0bf0b0db3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp index a88449e9ee490..fc6a50e66169e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp index b3c93c9f4abd4..5a63c4e89e7e7 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2017, 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp index 520df4d01289d..7dfe55e46430d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2017, 2020, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.cpp b/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.cpp index 3ea4e17ca60fd..d1409851e3ec5 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.cpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2017, 2019, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.hpp b/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.hpp index 489be9723dd83..ce1384a836d80 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.hpp @@ -1,5 +1,6 @@ /* * Copyright (c) 2017, 2022, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp index dfa12f4239a23..4ad92aa80a4ad 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp index 191a3f9046d10..4621ed6eea1fb 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, Amazon.com, Inc. or its affiliates. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp index afe110cf24d0c..0439fad44b56f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp @@ -1,6 +1,7 @@ /* - * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2016, 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/utilities/numberSeq.hpp b/src/hotspot/share/utilities/numberSeq.hpp index b0ced71e79f05..6e802aa598168 100644 --- a/src/hotspot/share/utilities/numberSeq.hpp +++ b/src/hotspot/share/utilities/numberSeq.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2019, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/gtest/gc/shenandoah/test_shenandoahNumberSeq.cpp b/test/hotspot/gtest/gc/shenandoah/test_shenandoahNumberSeq.cpp index e4f566ca36c8c..56c24788a82fa 100644 --- a/test/hotspot/gtest/gc/shenandoah/test_shenandoahNumberSeq.cpp +++ b/test/hotspot/gtest/gc/shenandoah/test_shenandoahNumberSeq.cpp @@ -1,5 +1,6 @@ /* - * Copyright (c) 2022, Amazon.com Inc. or its affiliates. All Rights Reserved. + * Copyright (c) 2022, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/gtest/gc/shenandoah/test_shenandoahOldHeuristic.cpp b/test/hotspot/gtest/gc/shenandoah/test_shenandoahOldHeuristic.cpp index d5470bae1146c..3665c56bd3f0e 100644 --- a/test/hotspot/gtest/gc/shenandoah/test_shenandoahOldHeuristic.cpp +++ b/test/hotspot/gtest/gc/shenandoah/test_shenandoahOldHeuristic.cpp @@ -1,3 +1,26 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + #include "precompiled.hpp" #include "unittest.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" diff --git a/test/hotspot/jtreg/gc/TestAllocHumongousFragment.java b/test/hotspot/jtreg/gc/TestAllocHumongousFragment.java index 9a935809c06c1..9daebd64d1144 100644 --- a/test/hotspot/jtreg/gc/TestAllocHumongousFragment.java +++ b/test/hotspot/jtreg/gc/TestAllocHumongousFragment.java @@ -1,6 +1,7 @@ /* * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2016, 2018, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/TestAllocIntArrays.java b/test/hotspot/jtreg/gc/shenandoah/TestAllocIntArrays.java index 15d8d4683784b..250b2c847d5aa 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestAllocIntArrays.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestAllocIntArrays.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2016, 2018, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/TestAllocObjectArrays.java b/test/hotspot/jtreg/gc/shenandoah/TestAllocObjectArrays.java index f30fda698fa56..64c78b03a656d 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestAllocObjectArrays.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestAllocObjectArrays.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2016, 2018, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/TestAllocObjects.java b/test/hotspot/jtreg/gc/shenandoah/TestAllocObjects.java index 71144e649d757..e8569b3bc7cb0 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestAllocObjects.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestAllocObjects.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2016, 2018, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/TestArrayCopyCheckCast.java b/test/hotspot/jtreg/gc/shenandoah/TestArrayCopyCheckCast.java index 1a89ba41bfa5b..c370d03a55e5d 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestArrayCopyCheckCast.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestArrayCopyCheckCast.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/TestArrayCopyStress.java b/test/hotspot/jtreg/gc/shenandoah/TestArrayCopyStress.java index 7ebcd964fe83d..fd6e14145e4d8 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestArrayCopyStress.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestArrayCopyStress.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/TestDynamicSoftMaxHeapSize.java b/test/hotspot/jtreg/gc/shenandoah/TestDynamicSoftMaxHeapSize.java index 97a425d2e22dc..7a53796c5f0a6 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestDynamicSoftMaxHeapSize.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestDynamicSoftMaxHeapSize.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2020, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/TestElasticTLAB.java b/test/hotspot/jtreg/gc/shenandoah/TestElasticTLAB.java index f46a419e3a6e2..764797459b9cb 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestElasticTLAB.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestElasticTLAB.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/TestEvilSyncBug.java b/test/hotspot/jtreg/gc/shenandoah/TestEvilSyncBug.java index f4132c114f303..aab34e0f6c0e2 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestEvilSyncBug.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestEvilSyncBug.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2016, 2020, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/TestGCThreadGroups.java b/test/hotspot/jtreg/gc/shenandoah/TestGCThreadGroups.java index 4a3cd80f13781..eb0d1ec2b467c 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestGCThreadGroups.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestGCThreadGroups.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2017, 2020, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/TestHeapUncommit.java b/test/hotspot/jtreg/gc/shenandoah/TestHeapUncommit.java index bd97e1f861ccb..f4639b2d03844 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestHeapUncommit.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestHeapUncommit.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2017, 2018, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/TestHumongousThreshold.java b/test/hotspot/jtreg/gc/shenandoah/TestHumongousThreshold.java index 8f76cbff629ef..068b69f2855ee 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestHumongousThreshold.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestHumongousThreshold.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2017, 2018, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/TestJcmdHeapDump.java b/test/hotspot/jtreg/gc/shenandoah/TestJcmdHeapDump.java index 0ec81ec6e1884..6f6aa63d0a070 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestJcmdHeapDump.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestJcmdHeapDump.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/TestLargeObjectAlignment.java b/test/hotspot/jtreg/gc/shenandoah/TestLargeObjectAlignment.java index 2ace318a299f4..7780d57b45e4b 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestLargeObjectAlignment.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestLargeObjectAlignment.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2016, 2018, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/TestLotsOfCycles.java b/test/hotspot/jtreg/gc/shenandoah/TestLotsOfCycles.java index ca32a1023ec4e..db6e520f00776 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestLotsOfCycles.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestLotsOfCycles.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2017, 2018, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/TestObjItrWithHeapDump.java b/test/hotspot/jtreg/gc/shenandoah/TestObjItrWithHeapDump.java index dea81f9754ca0..d8c471fb86830 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestObjItrWithHeapDump.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestObjItrWithHeapDump.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2019, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/TestPeriodicGC.java b/test/hotspot/jtreg/gc/shenandoah/TestPeriodicGC.java index 1eeb904a19a63..34c243780f310 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestPeriodicGC.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestPeriodicGC.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2017, 2020, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/TestReferenceRefersToShenandoah.java b/test/hotspot/jtreg/gc/shenandoah/TestReferenceRefersToShenandoah.java index 6fd758e72b0f9..74c8dec8cc628 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestReferenceRefersToShenandoah.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestReferenceRefersToShenandoah.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/TestReferenceShortcutCycle.java b/test/hotspot/jtreg/gc/shenandoah/TestReferenceShortcutCycle.java index 7522723d2afe3..bfd82376cd16d 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestReferenceShortcutCycle.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestReferenceShortcutCycle.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/TestRefprocSanity.java b/test/hotspot/jtreg/gc/shenandoah/TestRefprocSanity.java index ffb195c76f5b0..42e87adbbce41 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestRefprocSanity.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestRefprocSanity.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/TestRegionSampling.java b/test/hotspot/jtreg/gc/shenandoah/TestRegionSampling.java index 2ceb2e590adfa..20be9b7d9698a 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestRegionSampling.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestRegionSampling.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2017, 2018, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/TestRegionSamplingLogging.java b/test/hotspot/jtreg/gc/shenandoah/TestRegionSamplingLogging.java index 6c891d9a2dcbb..0017328b517dc 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestRegionSamplingLogging.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestRegionSamplingLogging.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Amazon.com, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/TestResizeTLAB.java b/test/hotspot/jtreg/gc/shenandoah/TestResizeTLAB.java index 896a869c5c932..076adb93f89cc 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestResizeTLAB.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestResizeTLAB.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2020, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/TestRetainObjects.java b/test/hotspot/jtreg/gc/shenandoah/TestRetainObjects.java index 90a28670b3797..36fbeac514dda 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestRetainObjects.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestRetainObjects.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2017, 2018, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/TestShenandoahLogRotation.java b/test/hotspot/jtreg/gc/shenandoah/TestShenandoahLogRotation.java index 9b9d633ef88da..b4e1953c606c1 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestShenandoahLogRotation.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestShenandoahLogRotation.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022 Amazon.com, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/TestSieveObjects.java b/test/hotspot/jtreg/gc/shenandoah/TestSieveObjects.java index bb63fb886628b..06389a67e518d 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestSieveObjects.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestSieveObjects.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2017, 2018, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/TestSmallHeap.java b/test/hotspot/jtreg/gc/shenandoah/TestSmallHeap.java index c629d038df5a0..b754005975187 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestSmallHeap.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestSmallHeap.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2017, 2018, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/TestStringDedup.java b/test/hotspot/jtreg/gc/shenandoah/TestStringDedup.java index 4356a80fb8757..f1991c0f50cb4 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestStringDedup.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestStringDedup.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2017, 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/TestStringDedupStress.java b/test/hotspot/jtreg/gc/shenandoah/TestStringDedupStress.java index c6a6ca31785d6..d835c096ffa5f 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestStringDedupStress.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestStringDedupStress.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2017, 2021, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/TestStringInternCleanup.java b/test/hotspot/jtreg/gc/shenandoah/TestStringInternCleanup.java index 98e6e75d064ba..4cc410efdb32d 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestStringInternCleanup.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestStringInternCleanup.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2017, 2018, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/TestVerifyJCStress.java b/test/hotspot/jtreg/gc/shenandoah/TestVerifyJCStress.java index bfc5c24ebea56..cccf5a7527746 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestVerifyJCStress.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestVerifyJCStress.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2017, 2020, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/TestVerifyLevels.java b/test/hotspot/jtreg/gc/shenandoah/TestVerifyLevels.java index df5d9c92586c3..750914a7d3686 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestVerifyLevels.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestVerifyLevels.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2017, 2018, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/TestWithLogLevel.java b/test/hotspot/jtreg/gc/shenandoah/TestWithLogLevel.java index f5650927c14a0..9f0fa13d18268 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestWithLogLevel.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestWithLogLevel.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2017, 2018, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/TestWrongArrayMember.java b/test/hotspot/jtreg/gc/shenandoah/TestWrongArrayMember.java index 7045a85b2eccb..6373136079c11 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestWrongArrayMember.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestWrongArrayMember.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/compiler/TestClone.java b/test/hotspot/jtreg/gc/shenandoah/compiler/TestClone.java index 4035a20c02d4e..2b1340890e128 100644 --- a/test/hotspot/jtreg/gc/shenandoah/compiler/TestClone.java +++ b/test/hotspot/jtreg/gc/shenandoah/compiler/TestClone.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2019, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/compiler/TestReferenceCAS.java b/test/hotspot/jtreg/gc/shenandoah/compiler/TestReferenceCAS.java index 5ec6f95cb8645..277da9eeb6307 100644 --- a/test/hotspot/jtreg/gc/shenandoah/compiler/TestReferenceCAS.java +++ b/test/hotspot/jtreg/gc/shenandoah/compiler/TestReferenceCAS.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2016, 2018, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/generational/TestCLIModeGenerational.java b/test/hotspot/jtreg/gc/shenandoah/generational/TestCLIModeGenerational.java index dfb62d680ae82..dd9210aaaf21f 100644 --- a/test/hotspot/jtreg/gc/shenandoah/generational/TestCLIModeGenerational.java +++ b/test/hotspot/jtreg/gc/shenandoah/generational/TestCLIModeGenerational.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, Amazon, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/generational/TestConcurrentEvac.java b/test/hotspot/jtreg/gc/shenandoah/generational/TestConcurrentEvac.java index 4111c5eac5428..a8879ea31a1ba 100644 --- a/test/hotspot/jtreg/gc/shenandoah/generational/TestConcurrentEvac.java +++ b/test/hotspot/jtreg/gc/shenandoah/generational/TestConcurrentEvac.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, Amazon, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/generational/TestSimpleGenerational.java b/test/hotspot/jtreg/gc/shenandoah/generational/TestSimpleGenerational.java index 38fe2e9f1a994..4a9d3efc7191c 100644 --- a/test/hotspot/jtreg/gc/shenandoah/generational/TestSimpleGenerational.java +++ b/test/hotspot/jtreg/gc/shenandoah/generational/TestSimpleGenerational.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, Amazon, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/jni/TestJNICritical.java b/test/hotspot/jtreg/gc/shenandoah/jni/TestJNICritical.java index 93eee9fbe9de8..62d9addeaa46d 100644 --- a/test/hotspot/jtreg/gc/shenandoah/jni/TestJNICritical.java +++ b/test/hotspot/jtreg/gc/shenandoah/jni/TestJNICritical.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2016, 2018, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/jni/TestJNIGlobalRefs.java b/test/hotspot/jtreg/gc/shenandoah/jni/TestJNIGlobalRefs.java index bd8b0bae92a6b..86d75bcbd2b12 100644 --- a/test/hotspot/jtreg/gc/shenandoah/jni/TestJNIGlobalRefs.java +++ b/test/hotspot/jtreg/gc/shenandoah/jni/TestJNIGlobalRefs.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/jni/TestPinnedGarbage.java b/test/hotspot/jtreg/gc/shenandoah/jni/TestPinnedGarbage.java index ad2838e940d66..507772f9b1849 100644 --- a/test/hotspot/jtreg/gc/shenandoah/jni/TestPinnedGarbage.java +++ b/test/hotspot/jtreg/gc/shenandoah/jni/TestPinnedGarbage.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/jvmti/TestHeapDump.java b/test/hotspot/jtreg/gc/shenandoah/jvmti/TestHeapDump.java index ed0d589f2fb0f..dec5efd185547 100644 --- a/test/hotspot/jtreg/gc/shenandoah/jvmti/TestHeapDump.java +++ b/test/hotspot/jtreg/gc/shenandoah/jvmti/TestHeapDump.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2017, 2020, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestChurnNotifications.java b/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestChurnNotifications.java index a42e41d6295db..f8119deee771c 100644 --- a/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestChurnNotifications.java +++ b/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestChurnNotifications.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestMemoryMXBeans.java b/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestMemoryMXBeans.java index 63746564a6af5..ac350448ba47d 100644 --- a/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestMemoryMXBeans.java +++ b/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestMemoryMXBeans.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2017, 2018, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestMemoryPools.java b/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestMemoryPools.java index b6ec62de3678c..50f710a92c0b5 100644 --- a/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestMemoryPools.java +++ b/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestMemoryPools.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2017, 2018, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestPauseNotifications.java b/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestPauseNotifications.java index 845c181d9d05c..7c8407bd361d0 100644 --- a/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestPauseNotifications.java +++ b/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestPauseNotifications.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/oom/TestClassLoaderLeak.java b/test/hotspot/jtreg/gc/shenandoah/oom/TestClassLoaderLeak.java index 62315af2c0524..95ba8516635d8 100644 --- a/test/hotspot/jtreg/gc/shenandoah/oom/TestClassLoaderLeak.java +++ b/test/hotspot/jtreg/gc/shenandoah/oom/TestClassLoaderLeak.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/oom/TestThreadFailure.java b/test/hotspot/jtreg/gc/shenandoah/oom/TestThreadFailure.java index 37ce09d1a2fcb..a5156a074104d 100644 --- a/test/hotspot/jtreg/gc/shenandoah/oom/TestThreadFailure.java +++ b/test/hotspot/jtreg/gc/shenandoah/oom/TestThreadFailure.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2018, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/shenandoah/options/TestModeUnlock.java b/test/hotspot/jtreg/gc/shenandoah/options/TestModeUnlock.java index 7b72e26929217..c0b7690d77846 100644 --- a/test/hotspot/jtreg/gc/shenandoah/options/TestModeUnlock.java +++ b/test/hotspot/jtreg/gc/shenandoah/options/TestModeUnlock.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2020, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/stress/gcbasher/TestGCBasherWithShenandoah.java b/test/hotspot/jtreg/gc/stress/gcbasher/TestGCBasherWithShenandoah.java index 3dc34c90584a8..bebdee40bab04 100644 --- a/test/hotspot/jtreg/gc/stress/gcbasher/TestGCBasherWithShenandoah.java +++ b/test/hotspot/jtreg/gc/stress/gcbasher/TestGCBasherWithShenandoah.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2016, 2018, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/stress/gclocker/TestGCLockerWithShenandoah.java b/test/hotspot/jtreg/gc/stress/gclocker/TestGCLockerWithShenandoah.java index 326b2f70b2e91..88573c884232b 100644 --- a/test/hotspot/jtreg/gc/stress/gclocker/TestGCLockerWithShenandoah.java +++ b/test/hotspot/jtreg/gc/stress/gclocker/TestGCLockerWithShenandoah.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2017, 2018, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/stress/gcold/TestGCOldWithShenandoah.java b/test/hotspot/jtreg/gc/stress/gcold/TestGCOldWithShenandoah.java index cf8ea24b13644..9aa4b70db8dd9 100644 --- a/test/hotspot/jtreg/gc/stress/gcold/TestGCOldWithShenandoah.java +++ b/test/hotspot/jtreg/gc/stress/gcold/TestGCOldWithShenandoah.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved. +* Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/stress/jfr/TestStressBigAllocationGCEventsWithShenandoah.java b/test/hotspot/jtreg/gc/stress/jfr/TestStressBigAllocationGCEventsWithShenandoah.java index 7c7e78378d388..966cd1ce0e363 100644 --- a/test/hotspot/jtreg/gc/stress/jfr/TestStressBigAllocationGCEventsWithShenandoah.java +++ b/test/hotspot/jtreg/gc/stress/jfr/TestStressBigAllocationGCEventsWithShenandoah.java @@ -1,5 +1,6 @@ /* - * Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2015, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/test/hotspot/jtreg/gc/stress/systemgc/TestSystemGCWithShenandoah.java b/test/hotspot/jtreg/gc/stress/systemgc/TestSystemGCWithShenandoah.java index eed284e19f180..2c410a591259d 100644 --- a/test/hotspot/jtreg/gc/stress/systemgc/TestSystemGCWithShenandoah.java +++ b/test/hotspot/jtreg/gc/stress/systemgc/TestSystemGCWithShenandoah.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2017, 2018, Red Hat, Inc. All rights reserved. + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it From 461f083162a8a9ea7adb8d6229b71124a98ef668 Mon Sep 17 00:00:00 2001 From: Aleksey Shipilev Date: Mon, 10 Apr 2023 17:08:21 +0000 Subject: [PATCH 210/254] Move affiliation and mode out of ShenandoahGenerationalMode Reviewed-by: kdnilsen, wkemper --- .../shenandoahAdaptiveHeuristics.cpp | 2 +- .../heuristics/shenandoahHeuristics.cpp | 15 ++--- .../heuristics/shenandoahHeuristics.hpp | 6 +- .../heuristics/shenandoahOldHeuristics.cpp | 4 +- .../mode/shenandoahGenerationalMode.cpp | 50 --------------- .../mode/shenandoahGenerationalMode.hpp | 18 ------ .../gc/shenandoah/shenandoahAffiliation.hpp | 62 +++++++++++++++++++ .../gc/shenandoah/shenandoahAllocRequest.hpp | 22 ++++--- .../gc/shenandoah/shenandoahCollectionSet.cpp | 4 +- .../gc/shenandoah/shenandoahConcurrentGC.cpp | 8 +-- .../shenandoah/shenandoahConcurrentMark.cpp | 16 ++--- .../shenandoah/shenandoahConcurrentMark.hpp | 8 +-- .../gc/shenandoah/shenandoahControlThread.cpp | 20 +++--- .../gc/shenandoah/shenandoahControlThread.hpp | 8 +-- .../gc/shenandoah/shenandoahDegeneratedGC.cpp | 8 +-- .../share/gc/shenandoah/shenandoahFreeSet.cpp | 31 +++++----- .../share/gc/shenandoah/shenandoahFreeSet.hpp | 2 +- .../share/gc/shenandoah/shenandoahFullGC.cpp | 22 +++---- .../gc/shenandoah/shenandoahGeneration.cpp | 12 ++-- .../gc/shenandoah/shenandoahGeneration.hpp | 15 ++--- .../shenandoah/shenandoahGenerationType.hpp | 48 ++++++++++++++ .../share/gc/shenandoah/shenandoahHeap.cpp | 37 ++++++----- .../share/gc/shenandoah/shenandoahHeap.hpp | 23 +++---- .../gc/shenandoah/shenandoahHeap.inline.hpp | 32 +++++----- .../gc/shenandoah/shenandoahHeapRegion.cpp | 34 ++++------ .../gc/shenandoah/shenandoahHeapRegion.hpp | 13 ++-- .../shenandoahHeapRegion.inline.hpp | 16 +++-- .../shenandoahHeapRegionCounters.cpp | 2 +- .../share/gc/shenandoah/shenandoahMark.cpp | 12 ++-- .../share/gc/shenandoah/shenandoahMark.hpp | 12 ++-- .../gc/shenandoah/shenandoahMark.inline.hpp | 12 ++-- .../shenandoah/shenandoahMarkingContext.cpp | 6 +- .../shenandoahMarkingContext.inline.hpp | 6 +- .../gc/shenandoah/shenandoahMmuTracker.cpp | 6 +- .../share/gc/shenandoah/shenandoahOldGC.cpp | 2 +- .../gc/shenandoah/shenandoahOldGeneration.cpp | 3 +- .../gc/shenandoah/shenandoahOopClosures.hpp | 8 +-- .../shenandoahOopClosures.inline.hpp | 4 +- .../shenandoahReferenceProcessor.cpp | 3 +- .../share/gc/shenandoah/shenandoahSTWMark.cpp | 12 ++-- .../shenandoah/shenandoahScanRemembered.cpp | 2 +- .../shenandoahScanRemembered.inline.hpp | 1 + .../shenandoah/shenandoahYoungGeneration.cpp | 2 +- 43 files changed, 337 insertions(+), 292 deletions(-) create mode 100644 src/hotspot/share/gc/shenandoah/shenandoahAffiliation.hpp create mode 100644 src/hotspot/share/gc/shenandoah/shenandoahGenerationType.hpp diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp index acc8a4820f1b0..e6d3a342aa11e 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp @@ -96,7 +96,7 @@ void ShenandoahAdaptiveHeuristics::choose_collection_set_from_regiondata(Shenand // more garbage. This represents one of the reasons why we keep looking at regions even after we decide, for example, // to exclude one of the regions because it might require evacuation of too much live data. bool is_generational = heap->mode()->is_generational(); - bool is_global = (_generation->generation_mode() == GLOBAL); + bool is_global = _generation->is_global(); size_t capacity = heap->young_generation()->max_capacity(); // cur_young_garbage represents the amount of memory to be reclaimed from young-gen. In the case that live objects diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp index 019ad0c054d70..3690458f6f190 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp @@ -101,7 +101,7 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec bool is_generational = heap->mode()->is_generational(); assert(collection_set->count() == 0, "Must be empty"); - assert(_generation->generation_mode() != OLD, "Old GC invokes ShenandoahOldHeuristics::choose_collection_set()"); + assert(!_generation->is_old(), "Old GC invokes ShenandoahOldHeuristics::choose_collection_set()"); // Check all pinned regions have updated status before choosing the collection set. heap->assert_pinned_region_status(); @@ -141,7 +141,7 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec immediate_garbage += garbage; region->make_trash_immediate(); } else { - assert (_generation->generation_mode() != OLD, "OLD is handled elsewhere"); + assert(!_generation->is_old(), "OLD is handled elsewhere"); live_memory += region->get_live_data_bytes(); // This is our candidate for later consideration. candidates[cand_idx]._region = region; @@ -369,15 +369,12 @@ double ShenandoahHeuristics::elapsed_cycle_time() const { } bool ShenandoahHeuristics::in_generation(ShenandoahHeapRegion* region) { - return ((_generation->generation_mode() == GLOBAL) - || (_generation->generation_mode() == YOUNG && region->affiliation() == YOUNG_GENERATION) - || (_generation->generation_mode() == OLD && region->affiliation() == OLD_GENERATION)); + return _generation->is_global() + || (_generation->is_young() && region->is_young()) + || (_generation->is_old() && region->is_old()); } size_t ShenandoahHeuristics::min_free_threshold() { - size_t min_free_threshold = - _generation->generation_mode() == GenerationMode::OLD - ? ShenandoahOldMinFreeThreshold - : ShenandoahMinFreeThreshold; + size_t min_free_threshold = _generation->is_old() ? ShenandoahOldMinFreeThreshold : ShenandoahMinFreeThreshold; return _generation->soft_max_capacity() / 100 * min_free_threshold; } diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp index 5e55ce13213f0..ffedff5e84625 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp @@ -75,11 +75,11 @@ class ShenandoahHeuristics : public CHeapObj { ShenandoahGeneration* _generation; - // if (_generation->generation_mode() == GLOBAL) _region_data represents + // if (_generation->type() == GLOBAL) _region_data represents // the results of most recently completed global marking pass - // if (_generation->generation_mode() == OLD) _region_data represents + // if (_generation->type() == OLD) _region_data represents // the results of most recently completed old-gen marking pass - // if (_generation->generation_mode() == YOUNG) _region_data represents + // if (_generation->type() == YOUNG) _region_data represents // the results of most recently completed young-gen marking pass // // Note that there is some redundancy represented in _region_data because diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp index 883632ca309e4..756aeaa0e8f5b 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp @@ -44,7 +44,7 @@ ShenandoahOldHeuristics::ShenandoahOldHeuristics(ShenandoahOldGeneration* genera _promotion_failed(false), _old_generation(generation) { - assert(_generation->generation_mode() == OLD, "This service only available for old-gc heuristics"); + assert(_generation->is_old(), "This service only available for old-gc heuristics"); } bool ShenandoahOldHeuristics::prime_collection_set(ShenandoahCollectionSet* collection_set) { @@ -212,7 +212,7 @@ void ShenandoahOldHeuristics::choose_collection_set(ShenandoahCollectionSet* col } void ShenandoahOldHeuristics::prepare_for_old_collections() { - assert(_generation->generation_mode() == OLD, "This service only available for old-gc heuristics"); + assert(_generation->is_old(), "This service only available for old-gc heuristics"); ShenandoahHeap* heap = ShenandoahHeap::heap(); size_t cand_idx = 0; diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp index a9d07efe884a2..4b7e0a3193880 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp @@ -26,7 +26,6 @@ #include "gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp" #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" #include "gc/shenandoah/mode/shenandoahGenerationalMode.hpp" -#include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "logging/log.hpp" #include "logging/logTag.hpp" #include "runtime/globals_extension.hpp" @@ -64,52 +63,3 @@ ShenandoahHeuristics* ShenandoahGenerationalMode::initialize_heuristics(Shenando return new ShenandoahAdaptiveHeuristics(generation); } - -const char* affiliation_name(oop ptr) { - ShenandoahHeap* heap = ShenandoahHeap::heap(); - assert(heap->is_in(ptr), "Oop must be in the heap."); - ShenandoahHeapRegion* region = heap->heap_region_containing(ptr); - return affiliation_name(region->affiliation()); -} - -const char affiliation_code(ShenandoahRegionAffiliation type) { - switch(type) { - case ShenandoahRegionAffiliation::FREE: - return 'F'; - case ShenandoahRegionAffiliation::YOUNG_GENERATION: - return 'Y'; - case ShenandoahRegionAffiliation::OLD_GENERATION: - return 'O'; - default: - ShouldNotReachHere(); - return 'X'; - } -} - -const char* affiliation_name(ShenandoahRegionAffiliation type) { - switch (type) { - case ShenandoahRegionAffiliation::FREE: - return "FREE"; - case ShenandoahRegionAffiliation::YOUNG_GENERATION: - return "YOUNG"; - case ShenandoahRegionAffiliation::OLD_GENERATION: - return "OLD"; - default: - ShouldNotReachHere(); - return nullptr; - } -} - -const char* generation_name(GenerationMode mode) { - switch (mode) { - case GenerationMode::GLOBAL: - return "Global"; - case GenerationMode::OLD: - return "Old"; - case GenerationMode::YOUNG: - return "Young"; - default: - ShouldNotReachHere(); - return nullptr; - } -} diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp index 5cf4fdaaa7979..71327bd5189af 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp @@ -26,24 +26,6 @@ #define SHARE_GC_SHENANDOAH_MODE_SHENANDOAHGENERATIONALMODE_HPP #include "gc/shenandoah/mode/shenandoahMode.hpp" -#include "oops/oopsHierarchy.hpp" - -enum GenerationMode { - YOUNG, - OLD, - GLOBAL -}; - -enum ShenandoahRegionAffiliation { - FREE, - YOUNG_GENERATION, - OLD_GENERATION -}; - -const char* affiliation_name(oop ptr); -const char* affiliation_name(ShenandoahRegionAffiliation type); -const char affiliation_code(ShenandoahRegionAffiliation type); -const char* generation_name(GenerationMode mode); class ShenandoahGenerationalMode : public ShenandoahMode { public: diff --git a/src/hotspot/share/gc/shenandoah/shenandoahAffiliation.hpp b/src/hotspot/share/gc/shenandoah/shenandoahAffiliation.hpp new file mode 100644 index 0000000000000..cacc62b78bfb4 --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahAffiliation.hpp @@ -0,0 +1,62 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHAFFILIATION_HPP +#define SHARE_GC_SHENANDOAH_SHENANDOAHAFFILIATION_HPP + +enum ShenandoahAffiliation { + FREE, + YOUNG_GENERATION, + OLD_GENERATION, +}; + +inline const char* shenandoah_affiliation_code(ShenandoahAffiliation type) { + switch(type) { + case FREE: + return "F"; + case YOUNG_GENERATION: + return "Y"; + case OLD_GENERATION: + return "O"; + default: + ShouldNotReachHere(); + return "?"; + } +} + +inline const char* shenandoah_affiliation_name(ShenandoahAffiliation type) { + switch (type) { + case FREE: + return "FREE"; + case YOUNG_GENERATION: + return "YOUNG"; + case OLD_GENERATION: + return "OLD"; + default: + ShouldNotReachHere(); + return "?"; + } +} + +#endif // SHARE_GC_SHENANDOAH_SHENANDOAHAFFILIATION_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahAllocRequest.hpp b/src/hotspot/share/gc/shenandoah/shenandoahAllocRequest.hpp index f05057c88df14..6f2dc92937109 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahAllocRequest.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahAllocRequest.hpp @@ -26,8 +26,8 @@ #ifndef SHARE_GC_SHENANDOAH_SHENANDOAHALLOCREQUEST_HPP #define SHARE_GC_SHENANDOAH_SHENANDOAHALLOCREQUEST_HPP +#include "gc/shenandoah/shenandoahAffiliation.hpp" #include "memory/allocation.hpp" -#include "gc/shenandoah/mode/shenandoahGenerationalMode.hpp" class ShenandoahAllocRequest : StackObj { public: @@ -63,12 +63,12 @@ class ShenandoahAllocRequest : StackObj { size_t _requested_size; size_t _actual_size; Type _alloc_type; - ShenandoahRegionAffiliation const _affiliation; + ShenandoahAffiliation const _affiliation; #ifdef ASSERT bool _actual_size_set; #endif - ShenandoahAllocRequest(size_t _min_size, size_t _requested_size, Type _alloc_type, ShenandoahRegionAffiliation affiliation) : + ShenandoahAllocRequest(size_t _min_size, size_t _requested_size, Type _alloc_type, ShenandoahAffiliation affiliation) : _min_size(_min_size), _requested_size(_requested_size), _actual_size(0), _alloc_type(_alloc_type), _affiliation(affiliation) #ifdef ASSERT @@ -78,23 +78,23 @@ class ShenandoahAllocRequest : StackObj { public: static inline ShenandoahAllocRequest for_tlab(size_t min_size, size_t requested_size) { - return ShenandoahAllocRequest(min_size, requested_size, _alloc_tlab, ShenandoahRegionAffiliation::YOUNG_GENERATION); + return ShenandoahAllocRequest(min_size, requested_size, _alloc_tlab, ShenandoahAffiliation::YOUNG_GENERATION); } static inline ShenandoahAllocRequest for_gclab(size_t min_size, size_t requested_size) { - return ShenandoahAllocRequest(min_size, requested_size, _alloc_gclab, ShenandoahRegionAffiliation::YOUNG_GENERATION); + return ShenandoahAllocRequest(min_size, requested_size, _alloc_gclab, ShenandoahAffiliation::YOUNG_GENERATION); } static inline ShenandoahAllocRequest for_plab(size_t min_size, size_t requested_size) { - return ShenandoahAllocRequest(min_size, requested_size, _alloc_plab, ShenandoahRegionAffiliation::OLD_GENERATION); + return ShenandoahAllocRequest(min_size, requested_size, _alloc_plab, ShenandoahAffiliation::OLD_GENERATION); } - static inline ShenandoahAllocRequest for_shared_gc(size_t requested_size, ShenandoahRegionAffiliation affiliation) { + static inline ShenandoahAllocRequest for_shared_gc(size_t requested_size, ShenandoahAffiliation affiliation) { return ShenandoahAllocRequest(0, requested_size, _alloc_shared_gc, affiliation); } static inline ShenandoahAllocRequest for_shared(size_t requested_size) { - return ShenandoahAllocRequest(0, requested_size, _alloc_shared, ShenandoahRegionAffiliation::YOUNG_GENERATION); + return ShenandoahAllocRequest(0, requested_size, _alloc_shared, ShenandoahAffiliation::YOUNG_GENERATION); } inline size_t size() { @@ -180,9 +180,13 @@ class ShenandoahAllocRequest : StackObj { return _affiliation == YOUNG_GENERATION; } - ShenandoahRegionAffiliation affiliation() const { + ShenandoahAffiliation affiliation() const { return _affiliation; } + + const char* affiliation_name() const { + return shenandoah_affiliation_name(_affiliation); + } }; #endif // SHARE_GC_SHENANDOAH_SHENANDOAHALLOCREQUEST_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp index 59cfbf8f6236e..67222fe82d4dc 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp @@ -91,13 +91,13 @@ void ShenandoahCollectionSet::add_region(ShenandoahHeapRegion* r) { _cset_map[r->index()] = 1; - if (r->affiliation() == YOUNG_GENERATION) { + if (r->is_young()) { _young_region_count++; _young_bytes_to_evacuate += r->get_live_data_bytes(); if (r->age() >= InitialTenuringThreshold) { _young_bytes_to_promote += r->get_live_data_bytes(); } - } else if (r->affiliation() == OLD_GENERATION) { + } else if (r->is_old()) { _old_region_count++; _old_bytes_to_evacuate += r->get_live_data_bytes(); _old_garbage += r->garbage(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index 18d9869470ec1..bfb0bbd6b384c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -187,7 +187,7 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) { // Global marking has completed. We need to fill in any unmarked objects in the old generation // so that subsequent remembered set scans will not walk pointers into reclaimed memory. - if (!heap->cancelled_gc() && heap->mode()->is_generational() && _generation->generation_mode() == GLOBAL) { + if (!heap->cancelled_gc() && heap->mode()->is_generational() && _generation->is_global()) { entry_global_coalesce_and_fill(); } @@ -370,7 +370,7 @@ void ShenandoahConcurrentGC::entry_reset() { } void ShenandoahConcurrentGC::entry_scan_remembered_set() { - if (_generation->generation_mode() == YOUNG) { + if (_generation->is_young()) { ShenandoahHeap* const heap = ShenandoahHeap::heap(); TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); const char* msg = "Concurrent remembered set scanning"; @@ -622,7 +622,7 @@ void ShenandoahConcurrentGC::op_init_mark() { if (heap->mode()->is_generational()) { - if (_generation->generation_mode() == YOUNG || (_generation->generation_mode() == GLOBAL && ShenandoahVerify)) { + if (_generation->is_young() || (_generation->is_global() && ShenandoahVerify)) { // The current implementation of swap_remembered_set() copies the write-card-table // to the read-card-table. The remembered sets are also swapped for GLOBAL collections // so that the verifier works with the correct copy of the card table when verifying. @@ -630,7 +630,7 @@ void ShenandoahConcurrentGC::op_init_mark() { _generation->swap_remembered_set(); } - if (_generation->generation_mode() == GLOBAL) { + if (_generation->is_global()) { heap->cancel_old_gc(); } else if (heap->is_concurrent_old_mark_in_progress()) { // Purge the SATB buffers, transferring any valid, old pointers to the diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp index 0cea8b47fea0e..11e6be2eb7b25 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp @@ -47,7 +47,7 @@ #include "runtime/continuation.hpp" #include "runtime/threads.hpp" -template +template class ShenandoahConcurrentMarkingTask : public WorkerTask { private: ShenandoahConcurrentMark* const _cm; @@ -96,7 +96,7 @@ class ShenandoahSATBAndRemarkThreadsClosure : public ThreadClosure { } }; -template +template class ShenandoahFinalMarkingTask : public WorkerTask { private: ShenandoahConcurrentMark* _cm; @@ -142,7 +142,7 @@ ShenandoahConcurrentMark::ShenandoahConcurrentMark(ShenandoahGeneration* generat ShenandoahMark(generation) {} // Mark concurrent roots during concurrent phases -template +template class ShenandoahMarkConcurrentRootsTask : public WorkerTask { private: SuspendibleThreadSetJoiner _sts_joiner; @@ -160,7 +160,7 @@ class ShenandoahMarkConcurrentRootsTask : public WorkerTask { void work(uint worker_id); }; -template +template ShenandoahMarkConcurrentRootsTask::ShenandoahMarkConcurrentRootsTask(ShenandoahObjToScanQueueSet* qs, ShenandoahObjToScanQueueSet* old, ShenandoahReferenceProcessor* rp, @@ -174,7 +174,7 @@ ShenandoahMarkConcurrentRootsTask::ShenandoahMarkConcurrentRootsTask assert(!ShenandoahHeap::heap()->has_forwarded_objects(), "Not expected"); } -template +template void ShenandoahMarkConcurrentRootsTask::work(uint worker_id) { ShenandoahConcurrentWorkerSession worker_session(worker_id); ShenandoahObjToScanQueue* q = _queue_set->queue(worker_id); @@ -190,7 +190,7 @@ void ShenandoahConcurrentMark::mark_concurrent_roots() { WorkerThreads* workers = heap->workers(); ShenandoahReferenceProcessor* rp = _generation->ref_processor(); _generation->reserve_task_queues(workers->active_workers()); - switch (_generation->generation_mode()) { + switch (_generation->type()) { case YOUNG: { ShenandoahMarkConcurrentRootsTask task(task_queues(), old_task_queues(), rp, ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers()); workers->run_task(&task); @@ -231,7 +231,7 @@ void ShenandoahConcurrentMark::concurrent_mark() { ShenandoahSATBMarkQueueSet& qset = ShenandoahBarrierSet::satb_mark_queue_set(); ShenandoahFlushSATBHandshakeClosure flush_satb(qset); for (uint flushes = 0; flushes < ShenandoahMaxSATBBufferFlushes; flushes++) { - switch (_generation->generation_mode()) { + switch (_generation->type()) { case YOUNG: { TaskTerminator terminator(nworkers, task_queues()); ShenandoahConcurrentMarkingTask task(this, &terminator); @@ -301,7 +301,7 @@ void ShenandoahConcurrentMark::finish_mark_work() { StrongRootsScope scope(nworkers); TaskTerminator terminator(nworkers, task_queues()); - switch (_generation->generation_mode()) { + switch (_generation->type()) { case YOUNG:{ ShenandoahFinalMarkingTask task(this, &terminator, ShenandoahStringDedup::is_enabled()); heap->workers()->run_task(&task); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp index e74e0e6e7f9fc..6f9b402cdd144 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp @@ -28,15 +28,15 @@ #include "gc/shenandoah/shenandoahMark.hpp" -template +template class ShenandoahConcurrentMarkingTask; -template +template class ShenandoahFinalMarkingTask; class ShenandoahGeneration; class ShenandoahConcurrentMark: public ShenandoahMark { - template friend class ShenandoahConcurrentMarkingTask; - template friend class ShenandoahFinalMarkingTask; + template friend class ShenandoahConcurrentMarkingTask; + template friend class ShenandoahFinalMarkingTask; public: ShenandoahConcurrentMark(ShenandoahGeneration* generation); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index d744b418506bc..704b7060e5ca8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -61,7 +61,7 @@ ShenandoahControlThread::ShenandoahControlThread() : _regulator_lock(Mutex::nosafepoint - 2, "ShenandoahRegulatorGC_lock", true), _periodic_task(this), _requested_gc_cause(GCCause::_no_cause_specified), - _requested_generation(GenerationMode::GLOBAL), + _requested_generation(ShenandoahGenerationType::GLOBAL), _degen_point(ShenandoahGC::_degenerated_outside_cycle), _degen_generation(nullptr), _allocs_seen(0), @@ -93,7 +93,7 @@ void ShenandoahControlThread::run_service() { ShenandoahHeap* heap = ShenandoahHeap::heap(); GCMode default_mode = concurrent_normal; - GenerationMode generation = GLOBAL; + ShenandoahGenerationType generation = GLOBAL; GCCause::Cause default_cause = GCCause::_shenandoah_concurrent_gc; double last_shrink_time = os::elapsedTime(); @@ -148,7 +148,7 @@ void ShenandoahControlThread::run_service() { } ShenandoahHeuristics* heuristics = _degen_generation->heuristics(); - generation = _degen_generation->generation_mode(); + generation = _degen_generation->type(); bool old_gen_evacuation_failed = heap->clear_old_evacuation_failure(); // Do not bother with degenerated cycle if old generation evacuation failed. @@ -460,7 +460,7 @@ void ShenandoahControlThread::process_phase_timings(const ShenandoahHeap* heap) // +---> Global Degen +--------------------> Full <----+ // void ShenandoahControlThread::service_concurrent_normal_cycle( - const ShenandoahHeap* heap, const GenerationMode generation, GCCause::Cause cause) { + const ShenandoahHeap* heap, const ShenandoahGenerationType generation, GCCause::Cause cause) { GCIdMark gc_id_mark; switch (generation) { case YOUNG: { @@ -682,7 +682,7 @@ void ShenandoahControlThread::service_concurrent_cycle(const ShenandoahHeap* hea } else { assert(heap->cancelled_gc(), "Must have been cancelled"); check_cancellation_or_degen(gc.degen_point()); - assert(generation->generation_mode() != OLD, "Old GC takes a different control path"); + assert(!generation->is_old(), "Old GC takes a different control path"); // Concurrent young-gen collection degenerates to young // collection. Same for global collections. _degen_generation = generation; @@ -753,11 +753,11 @@ bool ShenandoahControlThread::service_stw_degenerated_cycle(GCCause::Cause cause gc.collect(cause); assert(heap->young_generation()->task_queues()->is_empty(), "Unexpected young generation marking tasks"); - if (_degen_generation->generation_mode() == GLOBAL) { + if (_degen_generation->is_global()) { assert(heap->old_generation()->task_queues()->is_empty(), "Unexpected old generation marking tasks"); assert(heap->global_generation()->task_queues()->is_empty(), "Unexpected global generation marking tasks"); } else { - assert(_degen_generation->generation_mode() == YOUNG, "Expected degenerated young cycle, if not global."); + assert(_degen_generation->is_young(), "Expected degenerated young cycle, if not global."); ShenandoahOldGeneration* old_generation = (ShenandoahOldGeneration*) heap->old_generation(); if (old_generation->state() == ShenandoahOldGeneration::BOOTSTRAPPING && !gc.upgraded_to_full()) { old_generation->transition_to(ShenandoahOldGeneration::MARKING); @@ -823,7 +823,7 @@ void ShenandoahControlThread::request_gc(GCCause::Cause cause) { } } -bool ShenandoahControlThread::request_concurrent_gc(GenerationMode generation) { +bool ShenandoahControlThread::request_concurrent_gc(ShenandoahGenerationType generation) { if (_preemption_requested.is_set() || _gc_requested.is_set() || ShenandoahHeap::heap()->cancelled_gc()) { // ignore subsequent requests from the heuristics return false; @@ -839,7 +839,7 @@ bool ShenandoahControlThread::request_concurrent_gc(GenerationMode generation) { } if (preempt_old_marking(generation)) { - log_info(gc)("Preempting old generation mark to allow %s GC.", generation_name(generation)); + log_info(gc)("Preempting old generation mark to allow %s GC.", shenandoah_generation_name(generation)); _requested_gc_cause = GCCause::_shenandoah_concurrent_gc; _requested_generation = generation; _preemption_requested.set(); @@ -859,7 +859,7 @@ void ShenandoahControlThread::notify_control_thread() { _control_lock.notify(); } -bool ShenandoahControlThread::preempt_old_marking(GenerationMode generation) { +bool ShenandoahControlThread::preempt_old_marking(ShenandoahGenerationType generation) { return generation == YOUNG && _allow_old_preemption.try_unset(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp index b75795692d6bd..e8c500b7dcbf0 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp @@ -91,7 +91,7 @@ class ShenandoahControlThread: public ConcurrentGCThread { ShenandoahSharedFlag _do_counters_update; ShenandoahSharedFlag _force_counters_update; GCCause::Cause _requested_gc_cause; - GenerationMode _requested_generation; + ShenandoahGenerationType _requested_generation; ShenandoahGC::ShenandoahDegenPoint _degen_point; ShenandoahGeneration* _degen_generation; @@ -136,7 +136,7 @@ class ShenandoahControlThread: public ConcurrentGCThread { bool is_implicit_gc(GCCause::Cause cause) const; // Returns true if the old generation marking was interrupted to allow a young cycle. - bool preempt_old_marking(GenerationMode generation); + bool preempt_old_marking(ShenandoahGenerationType generation); // Returns true if the soft maximum heap has been changed using management APIs. bool check_soft_max_changed() const; @@ -158,7 +158,7 @@ class ShenandoahControlThread: public ConcurrentGCThread { void request_gc(GCCause::Cause cause); // Return true if the request to start a concurrent GC for the given generation succeeded. - bool request_concurrent_gc(GenerationMode generation); + bool request_concurrent_gc(ShenandoahGenerationType generation); void handle_counters_update(); void handle_force_counters_update(); @@ -173,7 +173,7 @@ class ShenandoahControlThread: public ConcurrentGCThread { bool in_graceful_shutdown(); void service_concurrent_normal_cycle(const ShenandoahHeap* heap, - const GenerationMode generation, + const ShenandoahGenerationType generation, GCCause::Cause cause); void service_concurrent_old_cycle(const ShenandoahHeap* heap, diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp index 843e0c8a451b0..1c3732b9cdc2a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp @@ -93,7 +93,7 @@ void ShenandoahDegenGC::op_degenerated() { #ifdef ASSERT if (heap->mode()->is_generational()) { - if (_generation->generation_mode() == GenerationMode::GLOBAL) { + if (_generation->is_global()) { // We can only get to a degenerated global cycle _after_ a concurrent global cycle // has been cancelled. In which case, we expect the concurrent global cycle to have // cancelled the old gc already. @@ -133,9 +133,9 @@ void ShenandoahDegenGC::op_degenerated() { // Note that we can only do this for "outside-cycle" degens, otherwise we would risk // changing the cycle parameters mid-cycle during concurrent -> degenerated handover. - heap->set_unload_classes((!heap->mode()->is_generational() || _generation->generation_mode() == GLOBAL) && _generation->heuristics()->can_unload_classes()); + heap->set_unload_classes((!heap->mode()->is_generational() || _generation->is_global()) && _generation->heuristics()->can_unload_classes()); - if (heap->mode()->is_generational() && (_generation->generation_mode() == YOUNG || (_generation->generation_mode() == GLOBAL && ShenandoahVerify))) { + if (heap->mode()->is_generational() && (_generation->is_young() || (_generation->is_global() && ShenandoahVerify))) { // Swap remembered sets for young, or if the verifier will run during a global collect _generation->swap_remembered_set(); } @@ -185,7 +185,7 @@ void ShenandoahDegenGC::op_degenerated() { case _degenerated_evac: - if (heap->mode()->is_generational() && _generation->generation_mode() == GLOBAL) { + if (heap->mode()->is_generational() && _generation->is_global()) { op_global_coalesce_and_fill(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index aeaf0bb67481c..e0126ba43f9c8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -25,6 +25,7 @@ #include "precompiled.hpp" #include "gc/shared/tlab_globals.hpp" +#include "gc/shenandoah/shenandoahAffiliation.hpp" #include "gc/shenandoah/shenandoahBarrierSet.hpp" #include "gc/shenandoah/shenandoahFreeSet.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" @@ -72,7 +73,7 @@ bool ShenandoahFreeSet::is_collector_free(size_t idx) const { // regions are not considered to reside within the is_collector_free range. // HeapWord* ShenandoahFreeSet::allocate_with_old_affiliation(ShenandoahAllocRequest& req, bool& in_new_region) { - ShenandoahRegionAffiliation affiliation = ShenandoahRegionAffiliation::OLD_GENERATION; + ShenandoahAffiliation affiliation = ShenandoahAffiliation::OLD_GENERATION; size_t rightmost = MAX2(_collector_rightmost, _mutator_rightmost); size_t leftmost = MIN2(_collector_leftmost, _mutator_leftmost); @@ -93,7 +94,7 @@ HeapWord* ShenandoahFreeSet::allocate_with_old_affiliation(ShenandoahAllocReques return nullptr; } -HeapWord* ShenandoahFreeSet::allocate_with_affiliation(ShenandoahRegionAffiliation affiliation, ShenandoahAllocRequest& req, bool& in_new_region) { +HeapWord* ShenandoahFreeSet::allocate_with_affiliation(ShenandoahAffiliation affiliation, ShenandoahAllocRequest& req, bool& in_new_region) { for (size_t c = _collector_rightmost + 1; c > _collector_leftmost; c--) { // size_t is unsigned, need to dodge underflow when _leftmost = 0 size_t idx = c - 1; @@ -107,7 +108,7 @@ HeapWord* ShenandoahFreeSet::allocate_with_affiliation(ShenandoahRegionAffiliati } } } - log_debug(gc, free)("Could not allocate collector region with affiliation: %s for request " PTR_FORMAT, affiliation_name(affiliation), p2i(&req)); + log_debug(gc, free)("Could not allocate collector region with affiliation: %s for request " PTR_FORMAT, shenandoah_affiliation_name(affiliation), p2i(&req)); return nullptr; } @@ -129,21 +130,21 @@ HeapWord* ShenandoahFreeSet::allocate_single(ShenandoahAllocRequest& req, bool& bool allow_new_region = true; if (_heap->mode()->is_generational()) { switch (req.affiliation()) { - case ShenandoahRegionAffiliation::OLD_GENERATION: + case ShenandoahAffiliation::OLD_GENERATION: // Note: unsigned result from adjusted_unaffiliated_regions() will never be less than zero, but it may equal zero. if (_heap->old_generation()->adjusted_unaffiliated_regions() <= 0) { allow_new_region = false; } break; - case ShenandoahRegionAffiliation::YOUNG_GENERATION: + case ShenandoahAffiliation::YOUNG_GENERATION: // Note: unsigned result from adjusted_unaffiliated_regions() will never be less than zero, but it may equal zero. if (_heap->young_generation()->adjusted_unaffiliated_regions() <= 0) { allow_new_region = false; } break; - case ShenandoahRegionAffiliation::FREE: + case ShenandoahAffiliation::FREE: default: ShouldNotReachHere(); break; @@ -156,7 +157,7 @@ HeapWord* ShenandoahFreeSet::allocate_single(ShenandoahAllocRequest& req, bool& // Try to allocate in the mutator view for (size_t idx = _mutator_leftmost; idx <= _mutator_rightmost; idx++) { ShenandoahHeapRegion* r = _heap->get_region(idx); - if (is_mutator_free(idx) && (allow_new_region || r->affiliation() != ShenandoahRegionAffiliation::FREE)) { + if (is_mutator_free(idx) && (allow_new_region || r->is_affiliated())) { // try_allocate_in() increases used if the allocation is successful. HeapWord* result = try_allocate_in(r, req, in_new_region); if (result != nullptr) { @@ -190,7 +191,7 @@ HeapWord* ShenandoahFreeSet::allocate_single(ShenandoahAllocRequest& req, bool& } else { // First try to fit into a region that is already in use in the same generation. HeapWord* result; - if (req.affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) { + if (req.is_old()) { // TODO: this is a work around to address a deficiency in FreeSet representation. A better solution fixes // the FreeSet implementation to deal more efficiently with old-gen regions as being in the "collector free set" result = allocate_with_old_affiliation(req, in_new_region); @@ -251,7 +252,7 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah return nullptr; } try_recycle_trashed(r); - if (r->affiliation() == ShenandoahRegionAffiliation::FREE) { + if (!r->is_affiliated()) { ShenandoahMarkingContext* const ctx = _heap->complete_marking_context(); r->set_affiliation(req.affiliation()); if (r->is_old()) { @@ -269,7 +270,7 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah assert(ctx->is_bitmap_clear_range(ctx->top_bitmap(r), r->end()), "Bitmap above top_bitmap() must be clear"); } else if (r->affiliation() != req.affiliation()) { assert(_heap->mode()->is_generational(), "Request for %s from %s region should only happen in generational mode.", - affiliation_name(req.affiliation()), affiliation_name(r->affiliation())); + req.affiliation_name(), r->affiliation_name()); return nullptr; } @@ -332,7 +333,7 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah // Account for the alignment padding size_t padding = (free - usable_free) * HeapWordSize; increase_used(padding); - assert(r->affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION, "All PLABs reside in old-gen"); + assert(r->is_old(), "All PLABs reside in old-gen"); _heap->old_generation()->increase_used(padding); // For verification consistency, we need to report this padding to _heap _heap->increase_used(padding); @@ -384,7 +385,7 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah // Account for the alignment padding size_t padding = (free - usable_free) * HeapWordSize; increase_used(padding); - assert(r->affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION, "All PLABs reside in old-gen"); + assert(r->is_old(), "All PLABs reside in old-gen"); _heap->old_generation()->increase_used(padding); // For verification consistency, we need to report this padding to _heap _heap->increase_used(padding); @@ -417,7 +418,7 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah // next evacuation pass. r->set_update_watermark(r->top()); generation->increase_used(size * HeapWordSize); - if (r->affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) { + if (r->is_old()) { assert(req.type() != ShenandoahAllocRequest::_alloc_gclab, "old-gen allocations use PLAB or shared allocation"); // for plabs, we'll sort the difference between evac and promotion usage when we retire the plab } @@ -493,7 +494,7 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req) { size_t words_size = req.size(); size_t num = ShenandoahHeapRegion::required_regions(words_size * HeapWordSize); - assert(req.affiliation() == ShenandoahRegionAffiliation::YOUNG_GENERATION, "Humongous regions always allocated in YOUNG"); + assert(req.is_young(), "Humongous regions always allocated in YOUNG"); ShenandoahGeneration* generation = _heap->generation_for(req.affiliation()); // Check if there are enough regions left to satisfy allocation. @@ -574,7 +575,7 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req) { // ctx->clear_bitmap(r); log_debug(gc, free)("NOT clearing bitmap for Humongous region [" PTR_FORMAT ", " PTR_FORMAT "], top_bitmap: " PTR_FORMAT " at transition from FREE to %s", - p2i(r->bottom()), p2i(r->end()), p2i(ctx->top_bitmap(r)), affiliation_name(req.affiliation())); + p2i(r->bottom()), p2i(r->end()), p2i(ctx->top_bitmap(r)), req.affiliation_name()); _mutator_free_bitmap.clear_bit(r->index()); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp index b0f449770cfb3..0e744d4e7021f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp @@ -50,7 +50,7 @@ class ShenandoahFreeSet : public CHeapObj { bool is_collector_free(size_t idx) const; HeapWord* try_allocate_in(ShenandoahHeapRegion* region, ShenandoahAllocRequest& req, bool& in_new_region); - HeapWord* allocate_with_affiliation(ShenandoahRegionAffiliation affiliation, ShenandoahAllocRequest& req, bool& in_new_region); + HeapWord* allocate_with_affiliation(ShenandoahAffiliation affiliation, ShenandoahAllocRequest& req, bool& in_new_region); HeapWord* allocate_with_old_affiliation(ShenandoahAllocRequest& req, bool& in_new_region); // While holding the heap lock, allocate memory for a single object which is to be entirely contained diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index f251860a3ccb5..ec91e2d6a8fda 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -452,7 +452,7 @@ class ShenandoahPrepareForGenerationalCompactionObjectClosure : public ObjectClo ShenandoahHeapRegion* _old_to_region; ShenandoahHeapRegion* _young_to_region; ShenandoahHeapRegion* _from_region; - ShenandoahRegionAffiliation _from_affiliation; + ShenandoahAffiliation _from_affiliation; HeapWord* _old_compact_point; HeapWord* _young_compact_point; uint _worker_id; @@ -479,13 +479,13 @@ class ShenandoahPrepareForGenerationalCompactionObjectClosure : public ObjectClo _from_region = from_region; _from_affiliation = from_region->affiliation(); if (_from_region->has_live()) { - if (_from_affiliation == ShenandoahRegionAffiliation::OLD_GENERATION) { + if (_from_affiliation == ShenandoahAffiliation::OLD_GENERATION) { if (_old_to_region == nullptr) { _old_to_region = from_region; _old_compact_point = from_region->bottom(); } } else { - assert(_from_affiliation == ShenandoahRegionAffiliation::YOUNG_GENERATION, "from_region must be OLD or YOUNG"); + assert(_from_affiliation == ShenandoahAffiliation::YOUNG_GENERATION, "from_region must be OLD or YOUNG"); if (_young_to_region == nullptr) { _young_to_region = from_region; _young_compact_point = from_region->bottom(); @@ -537,7 +537,7 @@ class ShenandoahPrepareForGenerationalCompactionObjectClosure : public ObjectClo uint object_age = p->age(); bool promote_object = false; - if ((_from_affiliation == ShenandoahRegionAffiliation::YOUNG_GENERATION) && + if ((_from_affiliation == ShenandoahAffiliation::YOUNG_GENERATION) && (from_region_age + object_age >= InitialTenuringThreshold)) { if ((_old_to_region != nullptr) && (_old_compact_point + obj_size > _old_to_region->end())) { finish_old_region(); @@ -559,7 +559,7 @@ class ShenandoahPrepareForGenerationalCompactionObjectClosure : public ObjectClo } } - if (promote_object || (_from_affiliation == ShenandoahRegionAffiliation::OLD_GENERATION)) { + if (promote_object || (_from_affiliation == ShenandoahAffiliation::OLD_GENERATION)) { assert(_old_to_region != nullptr, "_old_to_region should not be nullptr when evacuating to OLD region"); if (_old_compact_point + obj_size > _old_to_region->end()) { ShenandoahHeapRegion* new_to_region; @@ -594,7 +594,7 @@ class ShenandoahPrepareForGenerationalCompactionObjectClosure : public ObjectClo p->forward_to(cast_to_oop(_old_compact_point)); _old_compact_point += obj_size; } else { - assert(_from_affiliation == ShenandoahRegionAffiliation::YOUNG_GENERATION, + assert(_from_affiliation == ShenandoahAffiliation::YOUNG_GENERATION, "_from_region must be OLD_GENERATION or YOUNG_GENERATION"); assert(_young_to_region != nullptr, "_young_to_region should not be nullptr when compacting YOUNG _from_region"); @@ -750,7 +750,7 @@ void ShenandoahPrepareForCompactionTask::work(uint worker_id) { while (from_region != nullptr) { assert(is_candidate_region(from_region), "Sanity"); log_debug(gc)("Worker %u compacting %s Region " SIZE_FORMAT " which had used " SIZE_FORMAT " and %s live", - worker_id, affiliation_name(from_region->affiliation()), + worker_id, from_region->affiliation_name(), from_region->index(), from_region->used(), from_region->has_live()? "has": "does not have"); cl.set_from_region(from_region); if (from_region->has_live()) { @@ -885,17 +885,17 @@ class ShenandoahTrashImmediateGarbageClosure: public ShenandoahHeapRegionClosure if (!_ctx->is_marked(humongous_obj)) { assert(!r->has_live(), "Humongous Start %s Region " SIZE_FORMAT " is not marked, should not have live", - affiliation_name(r->affiliation()), r->index()); + r->affiliation_name(), r->index()); log_debug(gc)("Trashing immediate humongous region " SIZE_FORMAT " because not marked", r->index()); _heap->trash_humongous_region_at(r); } else { assert(r->has_live(), - "Humongous Start %s Region " SIZE_FORMAT " should have live", affiliation_name(r->affiliation()), r->index()); + "Humongous Start %s Region " SIZE_FORMAT " should have live", r->affiliation_name(), r->index()); } } else if (r->is_humongous_continuation()) { // If we hit continuation, the non-live humongous starts should have been trashed already assert(r->humongous_start_region()->has_live(), - "Humongous Continuation %s Region " SIZE_FORMAT " should have live", affiliation_name(r->affiliation()), r->index()); + "Humongous Continuation %s Region " SIZE_FORMAT " should have live", r->affiliation_name(), r->index()); } else if (r->is_regular()) { if (!r->has_live()) { log_debug(gc)("Trashing immediate regular region " SIZE_FORMAT " because has no live", r->index()); @@ -1390,7 +1390,7 @@ void ShenandoahFullGC::compact_humongous_objects() { new_obj->init_mark(); { - ShenandoahRegionAffiliation original_affiliation = r->affiliation(); + ShenandoahAffiliation original_affiliation = r->affiliation(); for (size_t c = old_start; c <= old_end; c++) { ShenandoahHeapRegion* r = heap->get_region(c); // Leave humongous region affiliation unchanged. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index 458482e1e0b06..d846321a34409 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -777,14 +777,14 @@ void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) { ShenandoahCollectionSet* collection_set = heap->collection_set(); assert(!heap->is_full_gc_in_progress(), "Only for concurrent and degenerated GC"); - assert(generation_mode() != OLD, "Only YOUNG and GLOBAL GC perform evacuations"); + assert(!is_old(), "Only YOUNG and GLOBAL GC perform evacuations"); { ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_update_region_states : ShenandoahPhaseTimings::degen_gc_final_update_region_states); ShenandoahFinalMarkUpdateRegionStateClosure cl(complete_marking_context()); parallel_heap_region_iterate(&cl); - if (generation_mode() == YOUNG) { + if (is_young()) { // We always need to update the watermark for old regions. If there // are mixed collections pending, we also need to synchronize the // pinned status for old regions. Since we are already visiting every @@ -842,7 +842,7 @@ bool ShenandoahGeneration::is_bitmap_clear() { size_t num_regions = heap->num_regions(); for (size_t idx = 0; idx < num_regions; idx++) { ShenandoahHeapRegion* r = heap->get_region(idx); - if (contains(r) && (r->affiliation() != FREE)) { + if (contains(r) && r->is_affiliated()) { if (heap->is_bitmap_slice_committed(r) && (context->top_at_mark_start(r) > r->bottom()) && !context->is_bitmap_clear_range(r->bottom(), r->end())) { return false; @@ -879,11 +879,11 @@ void ShenandoahGeneration::cancel_marking() { set_concurrent_mark_in_progress(false); } -ShenandoahGeneration::ShenandoahGeneration(GenerationMode generation_mode, +ShenandoahGeneration::ShenandoahGeneration(ShenandoahGenerationType type, uint max_workers, size_t max_capacity, size_t soft_max_capacity) : - _generation_mode(generation_mode), + _type(type), _task_queues(new ShenandoahObjToScanQueueSet(max_workers)), _ref_processor(new ShenandoahReferenceProcessor(MAX2(max_workers, 1U))), _collection_thread_time_s(0.0), @@ -915,7 +915,7 @@ ShenandoahObjToScanQueueSet* ShenandoahGeneration::old_gen_task_queues() const { } void ShenandoahGeneration::scan_remembered_set(bool is_concurrent) { - assert(generation_mode() == YOUNG, "Should only scan remembered set for young generation."); + assert(is_young(), "Should only scan remembered set for young generation."); ShenandoahHeap* const heap = ShenandoahHeap::heap(); uint nworkers = heap->workers()->active_workers(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index b8c2f8bf8be8a..e63edd667df0a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -27,7 +27,7 @@ #include "memory/allocation.hpp" #include "gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp" -#include "gc/shenandoah/mode/shenandoahGenerationalMode.hpp" +#include "gc/shenandoah/shenandoahGenerationType.hpp" #include "gc/shenandoah/shenandoahLock.hpp" #include "gc/shenandoah/shenandoahMarkingContext.hpp" @@ -35,10 +35,11 @@ class ShenandoahHeapRegion; class ShenandoahHeapRegionClosure; class ShenandoahReferenceProcessor; class ShenandoahHeap; +class ShenandoahMode; class ShenandoahGeneration : public CHeapObj { private: - GenerationMode const _generation_mode; + ShenandoahGenerationType const _type; // Marking task queues and completeness ShenandoahObjToScanQueueSet* _task_queues; @@ -70,14 +71,14 @@ class ShenandoahGeneration : public CHeapObj { size_t consumed_by_advance_promotion); public: - ShenandoahGeneration(GenerationMode generation_mode, uint max_workers, size_t max_capacity, size_t soft_max_capacity); + ShenandoahGeneration(ShenandoahGenerationType type, uint max_workers, size_t max_capacity, size_t soft_max_capacity); ~ShenandoahGeneration(); - bool is_young() const { return _generation_mode == YOUNG; } - bool is_old() const { return _generation_mode == OLD; } - bool is_global() const { return _generation_mode == GLOBAL; } + bool is_young() const { return _type == YOUNG; } + bool is_old() const { return _type == OLD; } + bool is_global() const { return _type == GLOBAL; } - inline GenerationMode generation_mode() const { return _generation_mode; } + inline ShenandoahGenerationType type() const { return _type; } inline ShenandoahHeuristics* heuristics() const { return _heuristics; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationType.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationType.hpp new file mode 100644 index 0000000000000..abf1b0b0f42dd --- /dev/null +++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationType.hpp @@ -0,0 +1,48 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_GC_SHENANDOAH_SHENANDOAHGENERATIONTYPE_HPP +#define SHARE_GC_SHENANDOAH_SHENANDOAHGENERATIONTYPE_HPP + +enum ShenandoahGenerationType { + YOUNG, + OLD, + GLOBAL +}; + +inline const char* shenandoah_generation_name(ShenandoahGenerationType mode) { + switch (mode) { + case GLOBAL: + return "Global"; + case OLD: + return "Old"; + case YOUNG: + return "Young"; + default: + ShouldNotReachHere(); + return "?"; + } +} + +#endif // SHARE_GC_SHENANDOAH_SHENANDOAHGENERATIONTYPE_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 97892451b5c23..77fac13ee3d3d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -374,7 +374,7 @@ jint ShenandoahHeap::initialize() { _regions[i] = r; assert(!collection_set()->is_in(i), "New region should not be in collection set"); - _affiliations[i] = ShenandoahRegionAffiliation::FREE; + _affiliations[i] = ShenandoahAffiliation::FREE; } // Initialize to complete @@ -450,7 +450,7 @@ jint ShenandoahHeap::initialize() { } size_t ShenandoahHeap::max_size_for(ShenandoahGeneration* generation) const { - switch (generation->generation_mode()) { + switch (generation->type()) { case YOUNG: return _generation_sizer.max_young_size(); case OLD: return max_capacity() - _generation_sizer.min_young_size(); case GLOBAL: return max_capacity(); @@ -461,7 +461,7 @@ size_t ShenandoahHeap::max_size_for(ShenandoahGeneration* generation) const { } size_t ShenandoahHeap::min_size_for(ShenandoahGeneration* generation) const { - switch (generation->generation_mode()) { + switch (generation->type()) { case YOUNG: return _generation_sizer.min_young_size(); case OLD: return max_capacity() - _generation_sizer.max_young_size(); case GLOBAL: return min_capacity(); @@ -689,7 +689,7 @@ bool ShenandoahHeap::is_old_bitmap_stable() const { } bool ShenandoahHeap::is_gc_generation_young() const { - return _gc_generation != nullptr && _gc_generation->generation_mode() == YOUNG; + return _gc_generation != nullptr && _gc_generation->is_young(); } size_t ShenandoahHeap::used() const { @@ -1294,7 +1294,7 @@ HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req if (!try_smaller_lab_size) { result = (allow_allocation)? _free_set->allocate(req, in_new_region): nullptr; if (result != nullptr) { - if (req.affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) { + if (req.is_old()) { ShenandoahThreadLocalData::reset_plab_promoted(thread); if (req.is_gc_alloc()) { if (req.type() == ShenandoahAllocRequest::_alloc_plab) { @@ -1339,7 +1339,7 @@ HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req } } else { // The allocation failed. If this was a plab allocation, We've already retired it and no longer have a plab. - if ((req.affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) && req.is_gc_alloc() && + if (req.is_old() && req.is_gc_alloc() && (req.type() == ShenandoahAllocRequest::_alloc_plab)) { // We don't need to disable PLAB promotions because there is no PLAB. We leave promotions enabled because // this allows the surrounding infrastructure to retry alloc_plab_slow() with a smaller PLAB size. @@ -1851,8 +1851,7 @@ void ShenandoahHeap::on_cycle_start(GCCause::Cause cause, ShenandoahGeneration* void ShenandoahHeap::on_cycle_end(ShenandoahGeneration* generation) { generation->heuristics()->record_cycle_end(); - if (mode()->is_generational() && - ((generation->generation_mode() == GLOBAL) || upgraded_to_full())) { + if (mode()->is_generational() && (generation->is_global() || upgraded_to_full())) { // If we just completed a GLOBAL GC, claim credit for completion of young-gen and old-gen GC as well young_generation()->heuristics()->record_cycle_end(); old_generation()->heuristics()->record_cycle_end(); @@ -2639,11 +2638,11 @@ class ShenandoahUpdateHeapRefsTask : public WorkerTask { log_debug(gc)("ShenandoahUpdateHeapRefsTask::do_work(%u) looking at region " SIZE_FORMAT, worker_id, r->index()); bool region_progress = false; if (r->is_active() && !r->is_cset()) { - if (!_heap->mode()->is_generational() || (r->affiliation() == ShenandoahRegionAffiliation::YOUNG_GENERATION)) { + if (!_heap->mode()->is_generational() || r->is_young()) { _heap->marked_object_oop_iterate(r, &cl, update_watermark); region_progress = true; - } else if (r->affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION) { - if (_heap->active_generation()->generation_mode() == GLOBAL) { + } else if (r->is_old()) { + if (_heap->active_generation()->is_global()) { // Note that GLOBAL collection is not as effectively balanced as young and mixed cycles. This is because // concurrent GC threads are parceled out entire heap regions of work at a time and there // is no "catchup phase" consisting of remembered set scanning, during which parcels of work are smaller @@ -2668,7 +2667,7 @@ class ShenandoahUpdateHeapRefsTask : public WorkerTask { assert(r->get_update_watermark() == r->bottom(), "%s Region " SIZE_FORMAT " is_active but not recognized as YOUNG or OLD so must be newly transitioned from FREE", - affiliation_name(r->affiliation()), r->index()); + r->affiliation_name(), r->index()); } } if (region_progress && ShenandoahPacing) { @@ -2680,7 +2679,7 @@ class ShenandoahUpdateHeapRefsTask : public WorkerTask { r = _regions->next(); } - if (_heap->mode()->is_generational() && (_heap->active_generation()->generation_mode() != GLOBAL)) { + if (_heap->mode()->is_generational() && !_heap->active_generation()->is_global()) { // Since this is generational and not GLOBAL, we have to process the remembered set. There's no remembered // set processing if not in generational mode or if GLOBAL mode. @@ -2693,7 +2692,7 @@ class ShenandoahUpdateHeapRefsTask : public WorkerTask { while (!_heap->check_cancelled_gc_and_yield(CONCURRENT) && _work_chunks->next(&assignment)) { // Keep grabbing next work chunk to process until finished, or asked to yield ShenandoahHeapRegion* r = assignment._r; - if (r->is_active() && !r->is_cset() && (r->affiliation() == ShenandoahRegionAffiliation::OLD_GENERATION)) { + if (r->is_active() && !r->is_cset() && r->is_old()) { HeapWord* start_of_range = r->bottom() + assignment._chunk_offset; HeapWord* end_of_range = r->get_update_watermark(); if (end_of_range > start_of_range + assignment._chunk_size) { @@ -3115,7 +3114,7 @@ void ShenandoahHeap::transfer_old_pointers_from_satb() { template<> void ShenandoahGenerationRegionClosure::heap_region_do(ShenandoahHeapRegion* region) { // Visit young and free regions - if (region->affiliation() != OLD_GENERATION) { + if (!region->is_old()) { _cl->heap_region_do(region); } } @@ -3123,7 +3122,7 @@ void ShenandoahGenerationRegionClosure::heap_region_do(ShenandoahHeapRegi template<> void ShenandoahGenerationRegionClosure::heap_region_do(ShenandoahHeapRegion* region) { // Visit old and free regions - if (region->affiliation() != YOUNG_GENERATION) { + if (!region->is_young()) { _cl->heap_region_do(region); } } @@ -3148,7 +3147,7 @@ void ShenandoahHeap::verify_rem_set_at_mark() { log_debug(gc)("Verifying remembered set at %s mark", doing_mixed_evacuations()? "mixed": "young"); - if (is_old_bitmap_stable() || active_generation()->generation_mode() == GLOBAL) { + if (is_old_bitmap_stable() || active_generation()->is_global()) { ctx = complete_marking_context(); } else { ctx = nullptr; @@ -3283,7 +3282,7 @@ void ShenandoahHeap::verify_rem_set_at_update_ref() { ShenandoahRegionIterator iterator; ShenandoahMarkingContext* ctx; - if (is_old_bitmap_stable() || active_generation()->generation_mode() == GLOBAL) { + if (is_old_bitmap_stable() || active_generation()->is_global()) { ctx = complete_marking_context(); } else { ctx = nullptr; @@ -3300,7 +3299,7 @@ void ShenandoahHeap::verify_rem_set_at_update_ref() { } } -ShenandoahGeneration* ShenandoahHeap::generation_for(ShenandoahRegionAffiliation affiliation) const { +ShenandoahGeneration* ShenandoahHeap::generation_for(ShenandoahAffiliation affiliation) const { if (!mode()->is_generational()) { return global_generation(); } else if (affiliation == YOUNG_GENERATION) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index b0a5d8ce459d3..0fe5db7923006 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -30,12 +30,12 @@ #include "gc/shared/markBitMap.hpp" #include "gc/shared/softRefPolicy.hpp" #include "gc/shared/collectedHeap.hpp" -#include "gc/shenandoah/mode/shenandoahGenerationalMode.hpp" #include "gc/shenandoah/shenandoahAsserts.hpp" #include "gc/shenandoah/shenandoahAllocRequest.hpp" #include "gc/shenandoah/shenandoahLock.hpp" #include "gc/shenandoah/shenandoahEvacOOMHandler.hpp" #include "gc/shenandoah/shenandoahEvacTracker.hpp" +#include "gc/shenandoah/shenandoahGenerationType.hpp" #include "gc/shenandoah/shenandoahMmuTracker.hpp" #include "gc/shenandoah/shenandoahPadding.hpp" #include "gc/shenandoah/shenandoahSharedVariables.hpp" @@ -69,6 +69,7 @@ class ShenandoahFreeSet; class ShenandoahConcurrentMark; class ShenandoahFullGC; class ShenandoahMonitoringSupport; +class ShenandoahMode; class ShenandoahPacer; class ShenandoahReferenceProcessor; class ShenandoahVerifier; @@ -118,7 +119,7 @@ class ShenandoahHeapRegionClosure : public StackObj { virtual bool is_thread_safe() { return false; } }; -template +template class ShenandoahGenerationRegionClosure : public ShenandoahHeapRegionClosure { public: explicit ShenandoahGenerationRegionClosure(ShenandoahHeapRegionClosure* cl) : _cl(cl) {} @@ -271,7 +272,7 @@ class ShenandoahHeap : public CollectedHeap { bool _heap_region_special; size_t _num_regions; ShenandoahHeapRegion** _regions; - uint8_t* _affiliations; // Holds array of enum ShenandoahRegionAffiliation, including FREE status in non-generational mode + uint8_t* _affiliations; // Holds array of enum ShenandoahAffiliation, including FREE status in non-generational mode ShenandoahRegionIterator _update_refs_iterator; public: @@ -547,7 +548,7 @@ class ShenandoahHeap : public CollectedHeap { ShenandoahYoungGeneration* young_generation() const { return _young_generation; } ShenandoahGeneration* global_generation() const { return _global_generation; } ShenandoahOldGeneration* old_generation() const { return _old_generation; } - ShenandoahGeneration* generation_for(ShenandoahRegionAffiliation affiliation) const; + ShenandoahGeneration* generation_for(ShenandoahAffiliation affiliation) const; const ShenandoahGenerationSizer* generation_sizer() const { return &_generation_sizer; } size_t max_size_for(ShenandoahGeneration* generation) const; @@ -612,8 +613,8 @@ class ShenandoahHeap : public CollectedHeap { void stw_process_weak_roots(bool full_gc); void stw_weak_refs(bool full_gc); - inline void assert_lock_for_affiliation(ShenandoahRegionAffiliation orig_affiliation, - ShenandoahRegionAffiliation new_affiliation); + inline void assert_lock_for_affiliation(ShenandoahAffiliation orig_affiliation, + ShenandoahAffiliation new_affiliation); // Heap iteration support void scan_roots_for_iteration(ShenandoahScanObjectStack* oop_stack, ObjectIterateScanRootClosure* oops); @@ -635,11 +636,11 @@ class ShenandoahHeap : public CollectedHeap { inline bool is_in_old(const void* p) const; inline bool is_old(oop pobj) const; - inline ShenandoahRegionAffiliation region_affiliation(const ShenandoahHeapRegion* r); - inline void set_affiliation(ShenandoahHeapRegion* r, ShenandoahRegionAffiliation new_affiliation); + inline ShenandoahAffiliation region_affiliation(const ShenandoahHeapRegion* r); + inline void set_affiliation(ShenandoahHeapRegion* r, ShenandoahAffiliation new_affiliation); - inline ShenandoahRegionAffiliation region_affiliation(size_t index); - inline void set_affiliation(size_t index, ShenandoahRegionAffiliation new_affiliation); + inline ShenandoahAffiliation region_affiliation(size_t index); + inline void set_affiliation(size_t index, ShenandoahAffiliation new_affiliation); bool requires_barriers(stackChunkOop obj) const override; @@ -781,7 +782,7 @@ class ShenandoahHeap : public CollectedHeap { ShenandoahEvacOOMHandler _oom_evac_handler; ShenandoahSharedFlag _old_gen_oom_evac; - inline oop try_evacuate_object(oop src, Thread* thread, ShenandoahHeapRegion* from_region, ShenandoahRegionAffiliation target_gen); + inline oop try_evacuate_object(oop src, Thread* thread, ShenandoahHeapRegion* from_region, ShenandoahAffiliation target_gen); void handle_old_evacuation(HeapWord* obj, size_t words, bool promotion); void handle_old_evacuation_failure(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp index 947ba01fb2fb4..696fb11b21b10 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp @@ -343,7 +343,7 @@ inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) { ShenandoahHeapRegion* r = heap_region_containing(p); assert(!r->is_humongous(), "never evacuate humongous objects"); - ShenandoahRegionAffiliation target_gen = r->affiliation(); + ShenandoahAffiliation target_gen = r->affiliation(); if (mode()->is_generational() && ShenandoahHeap::heap()->is_gc_generation_young() && target_gen == YOUNG_GENERATION) { markWord mark = p->mark(); @@ -368,7 +368,7 @@ inline oop ShenandoahHeap::evacuate_object(oop p, Thread* thread) { // try_evacuate_object registers the object and dirties the associated remembered set information when evacuating // to OLD_GENERATION. inline oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, ShenandoahHeapRegion* from_region, - ShenandoahRegionAffiliation target_gen) { + ShenandoahAffiliation target_gen) { bool alloc_from_lab = true; bool has_plab = false; HeapWord* copy = nullptr; @@ -576,13 +576,13 @@ inline bool ShenandoahHeap::is_in_active_generation(oop obj) const { size_t index = heap_region_containing(obj)->index(); switch (_affiliations[index]) { - case ShenandoahRegionAffiliation::FREE: + case ShenandoahAffiliation::FREE: // Free regions are in Old, Young, Global return true; - case ShenandoahRegionAffiliation::YOUNG_GENERATION: + case ShenandoahAffiliation::YOUNG_GENERATION: // Young regions are in young_generation and global_generation, not in old_generation return (active_generation() != (ShenandoahGeneration*) old_generation()); - case ShenandoahRegionAffiliation::OLD_GENERATION: + case ShenandoahAffiliation::OLD_GENERATION: // Old regions are in old_generation and global_generation, not in young_generation return (active_generation() != (ShenandoahGeneration*) young_generation()); default: @@ -592,23 +592,23 @@ inline bool ShenandoahHeap::is_in_active_generation(oop obj) const { } inline bool ShenandoahHeap::is_in_young(const void* p) const { - return is_in(p) && (_affiliations[heap_region_index_containing(p)] == ShenandoahRegionAffiliation::YOUNG_GENERATION); + return is_in(p) && (_affiliations[heap_region_index_containing(p)] == ShenandoahAffiliation::YOUNG_GENERATION); } inline bool ShenandoahHeap::is_in_old(const void* p) const { - return is_in(p) && (_affiliations[heap_region_index_containing(p)] == ShenandoahRegionAffiliation::OLD_GENERATION); + return is_in(p) && (_affiliations[heap_region_index_containing(p)] == ShenandoahAffiliation::OLD_GENERATION); } inline bool ShenandoahHeap::is_old(oop obj) const { return is_gc_generation_young() && is_in_old(obj); } -inline ShenandoahRegionAffiliation ShenandoahHeap::region_affiliation(const ShenandoahHeapRegion *r) { - return (ShenandoahRegionAffiliation) _affiliations[r->index()]; +inline ShenandoahAffiliation ShenandoahHeap::region_affiliation(const ShenandoahHeapRegion *r) { + return (ShenandoahAffiliation) _affiliations[r->index()]; } -inline void ShenandoahHeap::assert_lock_for_affiliation(ShenandoahRegionAffiliation orig_affiliation, - ShenandoahRegionAffiliation new_affiliation) { +inline void ShenandoahHeap::assert_lock_for_affiliation(ShenandoahAffiliation orig_affiliation, + ShenandoahAffiliation new_affiliation) { // A lock is required when changing from FREE to NON-FREE. Though it may be possible to elide the lock when // transitioning from in-use to FREE, the current implementation uses a lock for this transition. A lock is // not required to change from YOUNG to OLD (i.e. when promoting humongous region). @@ -623,24 +623,24 @@ inline void ShenandoahHeap::assert_lock_for_affiliation(ShenandoahRegionAffiliat // // Note: during full GC, all transitions between states are possible. During Full GC, we should be in a safepoint. - if ((orig_affiliation == ShenandoahRegionAffiliation::FREE) || (new_affiliation == ShenandoahRegionAffiliation::FREE)) { + if ((orig_affiliation == ShenandoahAffiliation::FREE) || (new_affiliation == ShenandoahAffiliation::FREE)) { extern bool _is_at_shenandoah_safepoint(); shenandoah_assert_heaplocked_or_fullgc_safepoint(); } } -inline void ShenandoahHeap::set_affiliation(ShenandoahHeapRegion* r, ShenandoahRegionAffiliation new_affiliation) { +inline void ShenandoahHeap::set_affiliation(ShenandoahHeapRegion* r, ShenandoahAffiliation new_affiliation) { #ifdef ASSERT assert_lock_for_affiliation(region_affiliation(r), new_affiliation); #endif _affiliations[r->index()] = (uint8_t) new_affiliation; } -inline ShenandoahRegionAffiliation ShenandoahHeap::region_affiliation(size_t index) { - return (ShenandoahRegionAffiliation) _affiliations[index]; +inline ShenandoahAffiliation ShenandoahHeap::region_affiliation(size_t index) { + return (ShenandoahAffiliation) _affiliations[index]; } -inline void ShenandoahHeap::set_affiliation(size_t index, ShenandoahRegionAffiliation new_affiliation) { +inline void ShenandoahHeap::set_affiliation(size_t index, ShenandoahAffiliation new_affiliation) { #ifdef ASSERT assert_lock_for_affiliation(region_affiliation(index), new_affiliation); #endif diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index 2dc88cdba2fd8..ca59f87af001f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -95,7 +95,7 @@ void ShenandoahHeapRegion::report_illegal_transition(const char *method) { fatal("%s", ss.freeze()); } -void ShenandoahHeapRegion::make_regular_allocation(ShenandoahRegionAffiliation affiliation) { +void ShenandoahHeapRegion::make_regular_allocation(ShenandoahAffiliation affiliation) { shenandoah_assert_heaplocked(); reset_age(); switch (_state) { @@ -172,7 +172,7 @@ void ShenandoahHeapRegion::make_humongous_start() { } } -void ShenandoahHeapRegion::make_humongous_start_bypass(ShenandoahRegionAffiliation affiliation) { +void ShenandoahHeapRegion::make_humongous_start_bypass(ShenandoahAffiliation affiliation) { shenandoah_assert_heaplocked(); assert (ShenandoahHeap::heap()->is_full_gc_in_progress(), "only for full GC"); set_affiliation(affiliation); @@ -203,7 +203,7 @@ void ShenandoahHeapRegion::make_humongous_cont() { } } -void ShenandoahHeapRegion::make_humongous_cont_bypass(ShenandoahRegionAffiliation affiliation) { +void ShenandoahHeapRegion::make_humongous_cont_bypass(ShenandoahAffiliation affiliation) { shenandoah_assert_heaplocked(); assert (ShenandoahHeap::heap()->is_full_gc_in_progress(), "only for full GC"); set_affiliation(affiliation); @@ -248,7 +248,7 @@ void ShenandoahHeapRegion::make_unpinned() { switch (_state) { case _pinned: - assert(affiliation() != FREE, "Pinned region should not be FREE"); + assert(is_affiliated(), "Pinned region should be affiliated"); set_state(_regular); return; case _regular: @@ -410,19 +410,7 @@ void ShenandoahHeapRegion::print_on(outputStream* st) const { ShouldNotReachHere(); } - switch (ShenandoahHeap::heap()->region_affiliation(this)) { - case ShenandoahRegionAffiliation::FREE: - st->print("|F"); - break; - case ShenandoahRegionAffiliation::YOUNG_GENERATION: - st->print("|Y"); - break; - case ShenandoahRegionAffiliation::OLD_GENERATION: - st->print("|O"); - break; - default: - ShouldNotReachHere(); - } + st->print("|%s", shenandoah_affiliation_code(affiliation())); #define SHR_PTR_FORMAT "%12" PRIxPTR @@ -935,15 +923,15 @@ size_t ShenandoahHeapRegion::pin_count() const { return Atomic::load(&_critical_pins); } -void ShenandoahHeapRegion::set_affiliation(ShenandoahRegionAffiliation new_affiliation) { +void ShenandoahHeapRegion::set_affiliation(ShenandoahAffiliation new_affiliation) { ShenandoahHeap* heap = ShenandoahHeap::heap(); - ShenandoahRegionAffiliation region_affiliation = heap->region_affiliation(this); + ShenandoahAffiliation region_affiliation = heap->region_affiliation(this); { ShenandoahMarkingContext* const ctx = heap->complete_marking_context(); log_debug(gc)("Setting affiliation of Region " SIZE_FORMAT " from %s to %s, top: " PTR_FORMAT ", TAMS: " PTR_FORMAT ", watermark: " PTR_FORMAT ", top_bitmap: " PTR_FORMAT, - index(), affiliation_name(region_affiliation), affiliation_name(new_affiliation), + index(), shenandoah_affiliation_name(region_affiliation), shenandoah_affiliation_name(new_affiliation), p2i(top()), p2i(ctx->top_at_mark_start(this)), p2i(_update_watermark), p2i(ctx->top_bitmap(this))); } @@ -970,11 +958,11 @@ void ShenandoahHeapRegion::set_affiliation(ShenandoahRegionAffiliation new_affil } log_trace(gc)("Changing affiliation of region %zu from %s to %s", - index(), affiliation_name(region_affiliation), affiliation_name(new_affiliation)); + index(), shenandoah_affiliation_name(region_affiliation), shenandoah_affiliation_name(new_affiliation)); - if (region_affiliation == ShenandoahRegionAffiliation::YOUNG_GENERATION) { + if (region_affiliation == ShenandoahAffiliation::YOUNG_GENERATION) { heap->young_generation()->decrement_affiliated_region_count(); - } else if (region_affiliation == ShenandoahRegionAffiliation::OLD_GENERATION) { + } else if (region_affiliation == ShenandoahAffiliation::OLD_GENERATION) { heap->old_generation()->decrement_affiliated_region_count(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp index 951c1fc7dbb07..20c9340699527 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp @@ -28,6 +28,7 @@ #include "gc/shared/gc_globals.hpp" #include "gc/shared/spaceDecorator.hpp" +#include "gc/shenandoah/shenandoahAffiliation.hpp" #include "gc/shenandoah/shenandoahAllocRequest.hpp" #include "gc/shenandoah/shenandoahAsserts.hpp" #include "gc/shenandoah/shenandoahHeap.hpp" @@ -169,13 +170,13 @@ class ShenandoahHeapRegion { } // Allowed transitions from the outside code: - void make_regular_allocation(ShenandoahRegionAffiliation affiliation); + void make_regular_allocation(ShenandoahAffiliation affiliation); void make_young_maybe(); void make_regular_bypass(); void make_humongous_start(); void make_humongous_cont(); - void make_humongous_start_bypass(ShenandoahRegionAffiliation affiliation); - void make_humongous_cont_bypass(ShenandoahRegionAffiliation affiliation); + void make_humongous_start_bypass(ShenandoahAffiliation affiliation); + void make_humongous_cont_bypass(ShenandoahAffiliation affiliation); void make_pinned(); void make_unpinned(); void make_cset(); @@ -202,6 +203,7 @@ class ShenandoahHeapRegion { bool is_pinned() const { return _state == _pinned || _state == _pinned_cset || _state == _pinned_humongous_start; } inline bool is_young() const; inline bool is_old() const; + inline bool is_affiliated() const; // Macro-properties: bool is_alloc_allowed() const { return is_empty() || is_regular() || _state == _pinned; } @@ -445,9 +447,10 @@ class ShenandoahHeapRegion { inline void set_update_watermark(HeapWord* w); inline void set_update_watermark_at_safepoint(HeapWord* w); - inline ShenandoahRegionAffiliation affiliation() const; + inline ShenandoahAffiliation affiliation() const; + inline const char* affiliation_name() const; - void set_affiliation(ShenandoahRegionAffiliation new_affiliation); + void set_affiliation(ShenandoahAffiliation new_affiliation); uint age() { return _age; } void increment_age() { _age++; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp index 6ed0a208c2603..88f84da09e351 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp @@ -160,7 +160,7 @@ inline void ShenandoahHeapRegion::internal_increase_live_data(size_t s) { size_t used_bytes = used(); assert(live_bytes <= used_bytes, "%s Region " SIZE_FORMAT " can't have more live data than used: " SIZE_FORMAT ", " SIZE_FORMAT " after adding " SIZE_FORMAT, - affiliation_name(affiliation()), index(), live_bytes, used_bytes, s * HeapWordSize); + affiliation_name(), index(), live_bytes, used_bytes, s * HeapWordSize); #endif } @@ -207,10 +207,14 @@ inline void ShenandoahHeapRegion::set_update_watermark_at_safepoint(HeapWord* w) _update_watermark = w; } -inline ShenandoahRegionAffiliation ShenandoahHeapRegion::affiliation() const { +inline ShenandoahAffiliation ShenandoahHeapRegion::affiliation() const { return ShenandoahHeap::heap()->region_affiliation(this); } +inline const char* ShenandoahHeapRegion::affiliation_name() const { + return shenandoah_affiliation_name(affiliation()); +} + inline void ShenandoahHeapRegion::clear_young_lab_flags() { _has_young_lab = false; } @@ -224,11 +228,15 @@ inline bool ShenandoahHeapRegion::has_young_lab_flag() { } inline bool ShenandoahHeapRegion::is_young() const { - return ShenandoahHeap::heap()->region_affiliation(this) == YOUNG_GENERATION; + return affiliation() == YOUNG_GENERATION; } inline bool ShenandoahHeapRegion::is_old() const { - return ShenandoahHeap::heap()->region_affiliation(this) == OLD_GENERATION; + return affiliation() == OLD_GENERATION; +} + +inline bool ShenandoahHeapRegion::is_affiliated() const { + return affiliation() != FREE; } #endif // SHARE_GC_SHENANDOAH_SHENANDOAHHEAPREGION_INLINE_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp index 810e3b816f251..f7534e2a7a151 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp @@ -156,7 +156,7 @@ static int encode_phase(ShenandoahHeap* heap) { } static int get_generation_shift(ShenandoahGeneration* generation) { - switch (generation->generation_mode()) { + switch (generation->type()) { case GLOBAL: return 0; case OLD: return 2; case YOUNG: return 4; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp index ef98f27464927..dd013fcfcb379 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp @@ -64,7 +64,7 @@ ShenandoahMark::ShenandoahMark(ShenandoahGeneration* generation) : _old_gen_task_queues(generation->old_gen_task_queues()) { } -template +template void ShenandoahMark::mark_loop_prework(uint w, TaskTerminator *t, ShenandoahReferenceProcessor *rp, StringDedup::Requests* const req, bool update_refs) { ShenandoahObjToScanQueue* q = get_queue(w); ShenandoahObjToScanQueue* old = get_old_queue(w); @@ -88,7 +88,7 @@ void ShenandoahMark::mark_loop_prework(uint w, TaskTerminator *t, ShenandoahRefe } template -void ShenandoahMark::mark_loop(GenerationMode generation, uint worker_id, TaskTerminator* terminator, ShenandoahReferenceProcessor *rp, StringDedup::Requests* const req) { +void ShenandoahMark::mark_loop(ShenandoahGenerationType generation, uint worker_id, TaskTerminator* terminator, ShenandoahReferenceProcessor *rp, StringDedup::Requests* const req) { bool update_refs = ShenandoahHeap::heap()->has_forwarded_objects(); switch (generation) { case YOUNG: @@ -107,8 +107,8 @@ void ShenandoahMark::mark_loop(GenerationMode generation, uint worker_id, TaskTe } } -void ShenandoahMark::mark_loop(GenerationMode generation, uint worker_id, TaskTerminator* terminator, ShenandoahReferenceProcessor *rp, - bool cancellable, StringDedupMode dedup_mode, StringDedup::Requests* const req) { +void ShenandoahMark::mark_loop(ShenandoahGenerationType generation, uint worker_id, TaskTerminator* terminator, ShenandoahReferenceProcessor *rp, + bool cancellable, StringDedupMode dedup_mode, StringDedup::Requests* const req) { if (cancellable) { switch(dedup_mode) { case NO_DEDUP: @@ -136,7 +136,7 @@ void ShenandoahMark::mark_loop(GenerationMode generation, uint worker_id, TaskTe } } -template +template void ShenandoahMark::mark_loop_work(T* cl, ShenandoahLiveData* live_data, uint worker_id, TaskTerminator *terminator, StringDedup::Requests* const req) { uintx stride = ShenandoahMarkLoopStride; @@ -145,7 +145,7 @@ void ShenandoahMark::mark_loop_work(T* cl, ShenandoahLiveData* live_data, uint w ShenandoahObjToScanQueue* q; ShenandoahMarkTask t; - assert(heap->active_generation()->generation_mode() == GENERATION, "Sanity"); + assert(heap->active_generation()->type() == GENERATION, "Sanity"); heap->active_generation()->ref_processor()->set_mark_closure(worker_id, cl); /* diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.hpp index d5e8a6f645653..cbdc767315422 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMark.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.hpp @@ -45,7 +45,7 @@ class ShenandoahMark: public StackObj { ShenandoahMark(ShenandoahGeneration* generation); public: - template + template static inline void mark_through_ref(T* p, ShenandoahObjToScanQueue* q, ShenandoahObjToScanQueue* old, ShenandoahMarkingContext* const mark_context, bool weak); // Loom support @@ -76,13 +76,13 @@ class ShenandoahMark: public StackObj { inline void count_liveness(ShenandoahLiveData* live_data, oop obj); - template + template void mark_loop_work(T* cl, ShenandoahLiveData* live_data, uint worker_id, TaskTerminator *t, StringDedup::Requests* const req); - template + template void mark_loop_prework(uint worker_id, TaskTerminator *terminator, ShenandoahReferenceProcessor *rp, StringDedup::Requests* const req, bool update_refs); - template + template static bool in_generation(oop obj); static void mark_ref(ShenandoahObjToScanQueue* q, @@ -93,10 +93,10 @@ class ShenandoahMark: public StackObj { inline void dedup_string(oop obj, StringDedup::Requests* const req); protected: template - void mark_loop(GenerationMode generation, uint worker_id, TaskTerminator* terminator, ShenandoahReferenceProcessor *rp, + void mark_loop(ShenandoahGenerationType generation, uint worker_id, TaskTerminator* terminator, ShenandoahReferenceProcessor *rp, StringDedup::Requests* const req); - void mark_loop(GenerationMode generation, uint worker_id, TaskTerminator* terminator, ShenandoahReferenceProcessor *rp, + void mark_loop(ShenandoahGenerationType generation, uint worker_id, TaskTerminator* terminator, ShenandoahReferenceProcessor *rp, bool cancellable, StringDedupMode dedup_mode, StringDedup::Requests* const req); }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp index 6101aa8c48216..1610936d1c8ae 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp @@ -115,7 +115,7 @@ inline void ShenandoahMark::count_liveness(ShenandoahLiveData* live_data, oop ob if (!region->is_humongous_start()) { assert(!region->is_humongous(), "Cannot have continuations here"); - assert(region->affiliation() != FREE, "Do not count live data within Free Regular Region " SIZE_FORMAT, region_idx); + assert(region->is_affiliated(), "Do not count live data within Free Regular Region " SIZE_FORMAT, region_idx); ShenandoahLiveData cur = live_data[region_idx]; size_t new_val = size + cur; if (new_val >= SHENANDOAH_LIVEDATA_MAX) { @@ -130,11 +130,11 @@ inline void ShenandoahMark::count_liveness(ShenandoahLiveData* live_data, oop ob shenandoah_assert_in_correct_region(nullptr, obj); size_t num_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize); - assert(region->affiliation() != FREE, "Do not count live data within FREE Humongous Start Region " SIZE_FORMAT, region_idx); + assert(region->is_affiliated(), "Do not count live data within FREE Humongous Start Region " SIZE_FORMAT, region_idx); for (size_t i = region_idx; i < region_idx + num_regions; i++) { ShenandoahHeapRegion* chain_reg = heap->get_region(i); assert(chain_reg->is_humongous(), "Expecting a humongous region"); - assert(chain_reg->affiliation() != FREE, "Do not count live data within FREE Humongous Continuation Region " SIZE_FORMAT, i); + assert(chain_reg->is_affiliated(), "Do not count live data within FREE Humongous Continuation Region " SIZE_FORMAT, i); chain_reg->increase_live_data_gc_words(chain_reg->used() >> LogHeapWordSize); } } @@ -237,7 +237,7 @@ inline void ShenandoahMark::do_chunked_array(ShenandoahObjToScanQueue* q, T* cl, array->oop_iterate_range(cl, from, to); } -template +template class ShenandoahSATBBufferClosure : public SATBBufferClosure { private: ShenandoahObjToScanQueue* _queue; @@ -262,7 +262,7 @@ class ShenandoahSATBBufferClosure : public SATBBufferClosure { } }; -template +template bool ShenandoahMark::in_generation(oop obj) { // Each in-line expansion of in_generation() resolves GENERATION at compile time. if (GENERATION == YOUNG) @@ -275,7 +275,7 @@ bool ShenandoahMark::in_generation(oop obj) { return false; } -template +template inline void ShenandoahMark::mark_through_ref(T *p, ShenandoahObjToScanQueue* q, ShenandoahObjToScanQueue* old, ShenandoahMarkingContext* const mark_context, bool weak) { T o = RawAccess<>::oop_load(p); if (!CompressedOops::is_null(o)) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.cpp index b7d503205b5c8..81abb2f4e5817 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.cpp @@ -41,7 +41,7 @@ bool ShenandoahMarkingContext::is_bitmap_clear() const { size_t num_regions = heap->num_regions(); for (size_t idx = 0; idx < num_regions; idx++) { ShenandoahHeapRegion* r = heap->get_region(idx); - if ((r->affiliation() != FREE) && heap->is_bitmap_slice_committed(r) && !is_bitmap_clear_range(r->bottom(), r->end())) { + if (r->is_affiliated() && heap->is_bitmap_slice_committed(r) && !is_bitmap_clear_range(r->bottom(), r->end())) { return false; } } @@ -83,9 +83,9 @@ void ShenandoahMarkingContext::clear_bitmap(ShenandoahHeapRegion* r) { HeapWord* top_bitmap = _top_bitmaps[r->index()]; log_debug(gc)("SMC:clear_bitmap for %s Region " SIZE_FORMAT ", top_bitmap: " PTR_FORMAT, - affiliation_name(r->affiliation()), r->index(), p2i(top_bitmap)); + r->affiliation_name(), r->index(), p2i(top_bitmap)); - if (r->affiliation() != FREE) { + if (r->is_affiliated()) { if (top_bitmap > bottom) { _mark_bit_map.clear_range_large(MemRegion(bottom, top_bitmap)); _top_bitmaps[r->index()] = bottom; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.inline.hpp index 2e99dd6bff4fe..cee8e1574600d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.inline.hpp @@ -75,7 +75,7 @@ inline bool ShenandoahMarkingContext::allocated_after_mark_start(const HeapWord* } inline void ShenandoahMarkingContext::capture_top_at_mark_start(ShenandoahHeapRegion *r) { - if (r->affiliation() != FREE) { + if (r->is_affiliated()) { size_t idx = r->index(); HeapWord* old_tams = _top_at_mark_starts_base[idx]; HeapWord* new_tams = r->top(); @@ -91,11 +91,11 @@ inline void ShenandoahMarkingContext::capture_top_at_mark_start(ShenandoahHeapRe idx, p2i(old_tams), p2i(new_tams)); log_debug(gc)("Capturing TAMS for %s Region " SIZE_FORMAT ", was: %llx, now: %llx", - affiliation_name(r->affiliation()), idx, (unsigned long long) old_tams, (unsigned long long) new_tams); + r->affiliation_name(), idx, (unsigned long long) old_tams, (unsigned long long) new_tams); if ((old_tams == r->bottom()) && (new_tams > old_tams)) { log_debug(gc)("Clearing mark bitmap for %s Region " SIZE_FORMAT " while capturing TAMS", - affiliation_name(r->affiliation()), idx); + r->affiliation_name(), idx); clear_bitmap(r); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp index 568f070b6df31..b9443f129239a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp @@ -259,7 +259,7 @@ bool ShenandoahGenerationSizer::transfer_capacity(ShenandoahGeneration* from, Sh } size_t regions_to_transfer = MAX2(1u, uint(double(available_regions) * _resize_increment)); - if (from->generation_mode() == YOUNG) { + if (from->is_young()) { regions_to_transfer = adjust_transfer_from_young(from, regions_to_transfer); } else { regions_to_transfer = adjust_transfer_to_young(to, regions_to_transfer); @@ -279,7 +279,7 @@ bool ShenandoahGenerationSizer::transfer_capacity(ShenandoahGeneration* from, Sh } size_t ShenandoahGenerationSizer::adjust_transfer_from_young(ShenandoahGeneration* from, size_t regions_to_transfer) const { - assert(from->generation_mode() == YOUNG, "Expect to transfer from young"); + assert(from->is_young(), "Expect to transfer from young"); size_t young_capacity_regions = from->max_capacity() / ShenandoahHeapRegion::region_size_bytes(); size_t new_young_regions = young_capacity_regions - regions_to_transfer; size_t minimum_young_regions = min_young_regions(); @@ -294,7 +294,7 @@ size_t ShenandoahGenerationSizer::adjust_transfer_from_young(ShenandoahGeneratio } size_t ShenandoahGenerationSizer::adjust_transfer_to_young(ShenandoahGeneration* to, size_t regions_to_transfer) const { - assert(to->generation_mode() == YOUNG, "Can only transfer between young and old."); + assert(to->is_young(), "Can only transfer between young and old."); size_t young_capacity_regions = to->max_capacity() / ShenandoahHeapRegion::region_size_bytes(); size_t new_young_regions = young_capacity_regions + regions_to_transfer; size_t maximum_young_regions = max_young_regions(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp index b74d837071e0c..5d84bd8e4aab8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp @@ -54,7 +54,7 @@ void ShenandoahOldGC::op_final_mark() { } if (!heap->cancelled_gc()) { - assert(_mark.generation()->generation_mode() == OLD, "Generation of Old-Gen GC should be OLD"); + assert(_mark.generation()->is_old(), "Generation of Old-Gen GC should be OLD"); _mark.finish_mark(); assert(!heap->cancelled_gc(), "STW mark cannot OOM"); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp index c4433d788bba5..b197e2bdf887b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp @@ -181,7 +181,8 @@ const char* ShenandoahOldGeneration::name() const { } bool ShenandoahOldGeneration::contains(ShenandoahHeapRegion* region) const { - return region->affiliation() != YOUNG_GENERATION; + // TODO: Should this be region->is_old() instead? + return !region->is_young(); } void ShenandoahOldGeneration::parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp index 1d7664205e9b9..4ab5863923f1f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp @@ -48,7 +48,7 @@ class ShenandoahMarkRefsSuperClosure : public MetadataVisitingOopIterateClosure bool _weak; protected: - template + template void work(T *p); public: @@ -72,7 +72,7 @@ class ShenandoahMarkUpdateRefsSuperClosure : public ShenandoahMarkRefsSuperClosu protected: ShenandoahHeap* const _heap; - template + template inline void work(T* p); public: @@ -83,7 +83,7 @@ class ShenandoahMarkUpdateRefsSuperClosure : public ShenandoahMarkRefsSuperClosu }; }; -template +template class ShenandoahMarkUpdateRefsClosure : public ShenandoahMarkUpdateRefsSuperClosure { private: template @@ -97,7 +97,7 @@ class ShenandoahMarkUpdateRefsClosure : public ShenandoahMarkUpdateRefsSuperClos virtual void do_oop(oop* p) { do_oop_work(p); } }; -template +template class ShenandoahMarkRefsClosure : public ShenandoahMarkRefsSuperClosure { private: template diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp index 5110ea1f6e351..6846dd8e09637 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp @@ -31,12 +31,12 @@ #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahMark.inline.hpp" -template +template inline void ShenandoahMarkRefsSuperClosure::work(T* p) { ShenandoahMark::mark_through_ref(p, _queue, _old_queue, _mark_context, _weak); } -template +template inline void ShenandoahMarkUpdateRefsSuperClosure::work(T* p) { // Update the location _heap->update_with_forwarded(p); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp b/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp index e6beb585cdf00..1a1e3bf2b023c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahReferenceProcessor.cpp @@ -27,7 +27,6 @@ #include "precompiled.hpp" #include "classfile/javaClasses.hpp" #include "gc/shared/workerThread.hpp" -#include "gc/shenandoah/mode/shenandoahGenerationalMode.hpp" #include "gc/shenandoah/shenandoahGeneration.hpp" #include "gc/shenandoah/shenandoahOopClosures.inline.hpp" #include "gc/shenandoah/shenandoahReferenceProcessor.hpp" @@ -388,7 +387,7 @@ bool ShenandoahReferenceProcessor::discover_reference(oop reference, ReferenceTy } log_trace(gc, ref)("Encountered Reference: " PTR_FORMAT " (%s, %s)", - p2i(reference), reference_type_name(type), affiliation_name(reference)); + p2i(reference), reference_type_name(type), ShenandoahHeap::heap()->heap_region_containing(reference)->affiliation_name()); uint worker_id = WorkerThread::worker_id(); _ref_proc_thread_locals->inc_encountered(type); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp index 246c11b2f572a..f4498e415b0c2 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp @@ -38,7 +38,7 @@ #include "gc/shenandoah/shenandoahSTWMark.hpp" #include "gc/shenandoah/shenandoahVerifier.hpp" -template +template class ShenandoahInitMarkRootsClosure : public OopClosure { private: ShenandoahObjToScanQueue* const _queue; @@ -54,13 +54,13 @@ class ShenandoahInitMarkRootsClosure : public OopClosure { void do_oop(oop* p) { do_oop_work(p); } }; -template +template ShenandoahInitMarkRootsClosure::ShenandoahInitMarkRootsClosure(ShenandoahObjToScanQueue* q) : _queue(q), _mark_context(ShenandoahHeap::heap()->marking_context()) { } -template +template template void ShenandoahInitMarkRootsClosure::do_oop_work(T* p) { // Only called from STW mark, should not be used to bootstrap old generation marking. @@ -117,7 +117,7 @@ void ShenandoahSTWMark::mark() { { // Mark - if (_generation->generation_mode() == YOUNG) { + if (_generation->is_young()) { // But only scan the remembered set for young generation. _generation->scan_remembered_set(false /* is_concurrent */); } @@ -138,7 +138,7 @@ void ShenandoahSTWMark::mark() { } void ShenandoahSTWMark::mark_roots(uint worker_id) { - switch (_generation->generation_mode()) { + switch (_generation->type()) { case GLOBAL: { ShenandoahInitMarkRootsClosure init_mark(task_queues()->queue(worker_id)); _root_scanner.roots_do(&init_mark, worker_id); @@ -160,7 +160,7 @@ void ShenandoahSTWMark::finish_mark(uint worker_id) { ShenandoahReferenceProcessor* rp = ShenandoahHeap::heap()->active_generation()->ref_processor(); StringDedup::Requests requests; - mark_loop(_generation->generation_mode(), + mark_loop(_generation->type(), worker_id, &_terminator, rp, false /* not cancellable */, ShenandoahStringDedup::is_enabled() ? ALWAYS_DEDUP : NO_DEDUP, &requests); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp index 2ba22813c215d..3476badcc793c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.cpp @@ -84,7 +84,7 @@ void ShenandoahScanRememberedTask::do_work(uint worker_id) { log_debug(gc)("ShenandoahScanRememberedTask::do_work(%u), processing slice of region " SIZE_FORMAT " at offset " SIZE_FORMAT ", size: " SIZE_FORMAT, worker_id, region->index(), assignment._chunk_offset, assignment._chunk_size); - if (region->affiliation() == OLD_GENERATION) { + if (region->is_old()) { size_t cluster_size = CardTable::card_size_in_words() * ShenandoahCardCluster::CardsPerCluster; size_t clusters = assignment._chunk_size / cluster_size; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp index c4a807fce3cc3..4a3fcb69acb9d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp @@ -34,6 +34,7 @@ #include "gc/shenandoah/shenandoahHeap.hpp" #include "gc/shenandoah/shenandoahHeapRegion.hpp" #include "gc/shenandoah/shenandoahScanRemembered.hpp" +#include "gc/shenandoah/mode/shenandoahMode.hpp" inline size_t ShenandoahDirectCardMarkRememberedSet::last_valid_index() const { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp index 4ad92aa80a4ad..632a3df67d52e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp @@ -55,7 +55,7 @@ void ShenandoahYoungGeneration::set_concurrent_mark_in_progress(bool in_progress bool ShenandoahYoungGeneration::contains(ShenandoahHeapRegion* region) const { // TODO: why not test for equals YOUNG_GENERATION? As written, returns true for regions that are FREE - return region->affiliation() != OLD_GENERATION; + return !region->is_old(); } void ShenandoahYoungGeneration::parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl) { From c9a6355d8669ce55f91e970bc93724563e89c2da Mon Sep 17 00:00:00 2001 From: Aleksey Shipilev Date: Tue, 11 Apr 2023 10:27:48 +0000 Subject: [PATCH 211/254] Cleanups, TODOs, asserts (part 1) Reviewed-by: wkemper, kdnilsen, ysr --- .../shenandoahBarrierSetAssembler_aarch64.cpp | 23 ++- .../shenandoahBarrierSetAssembler_aarch64.hpp | 4 +- .../shenandoahBarrierSetAssembler_ppc.cpp | 16 +- .../c1/shenandoahBarrierSetC1_x86.cpp | 2 +- .../shenandoahBarrierSetAssembler_x86.cpp | 49 +++--- .../shenandoahBarrierSetAssembler_x86.hpp | 4 +- .../shenandoah/c2/shenandoahBarrierSetC2.cpp | 45 +++-- .../shenandoahAdaptiveHeuristics.cpp | 30 ++-- .../shenandoahAggressiveHeuristics.cpp | 13 +- .../shenandoahCompactHeuristics.cpp | 4 +- .../heuristics/shenandoahHeuristics.cpp | 60 ++++--- .../heuristics/shenandoahHeuristics.hpp | 21 ++- .../heuristics/shenandoahOldHeuristics.cpp | 24 +-- .../shenandoahPassiveHeuristics.cpp | 2 +- .../heuristics/shenandoahStaticHeuristics.cpp | 4 +- .../mode/shenandoahGenerationalMode.cpp | 1 + .../share/gc/shenandoah/shenandoahAsserts.cpp | 3 +- .../gc/shenandoah/shenandoahBarrierSet.cpp | 1 + .../gc/shenandoah/shenandoahBarrierSet.hpp | 6 +- .../shenandoahBarrierSet.inline.hpp | 17 +- .../gc/shenandoah/shenandoahCardStats.cpp | 3 +- .../gc/shenandoah/shenandoahCardStats.hpp | 39 ++--- .../gc/shenandoah/shenandoahCardTable.cpp | 6 +- .../gc/shenandoah/shenandoahCollectionSet.cpp | 19 ++- .../gc/shenandoah/shenandoahCollectionSet.hpp | 13 +- .../gc/shenandoah/shenandoahConcurrentGC.cpp | 27 ++- .../gc/shenandoah/shenandoahConcurrentGC.hpp | 7 +- .../shenandoah/shenandoahConcurrentMark.cpp | 26 +-- .../shenandoah/shenandoahConcurrentMark.hpp | 2 +- .../gc/shenandoah/shenandoahControlThread.cpp | 154 +++++++++--------- .../gc/shenandoah/shenandoahControlThread.hpp | 4 +- .../gc/shenandoah/shenandoahDegeneratedGC.cpp | 18 +- .../gc/shenandoah/shenandoahEvacTracker.cpp | 13 +- .../gc/shenandoah/shenandoahEvacTracker.hpp | 17 +- .../share/gc/shenandoah/shenandoahFreeSet.cpp | 2 + .../share/gc/shenandoah/shenandoahFullGC.cpp | 98 ++++++----- .../gc/shenandoah/shenandoahGeneration.cpp | 29 ++-- .../gc/shenandoah/shenandoahGeneration.hpp | 14 +- .../shenandoah/shenandoahGlobalGeneration.hpp | 15 +- .../share/gc/shenandoah/shenandoahHeap.cpp | 4 +- .../gc/shenandoah/shenandoahHeap.inline.hpp | 1 - .../gc/shenandoah/shenandoahHeapRegion.cpp | 4 +- .../gc/shenandoah/shenandoahHeapRegion.hpp | 2 +- .../shenandoah/shenandoahScanRemembered.hpp | 6 +- .../shenandoahScanRemembered.inline.hpp | 8 +- 45 files changed, 461 insertions(+), 399 deletions(-) diff --git a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp index 36d9097dc0b44..3fc5e7bb0a7f1 100644 --- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp @@ -386,12 +386,9 @@ void ShenandoahBarrierSetAssembler::load_at(MacroAssembler* masm, DecoratorSet d void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register obj) { if (!ShenandoahHeap::heap()->mode()->is_generational()) { - return; + return; } - ShenandoahBarrierSet* ctbs = ShenandoahBarrierSet::barrier_set(); - CardTable* ct = ctbs->card_table(); - __ lsr(obj, obj, CardTable::card_shift()); assert(CardTable::dirty_card_val() == 0, "must be"); @@ -400,7 +397,7 @@ void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register o if (UseCondCardMark) { Label L_already_dirty; - __ ldrb(rscratch2, Address(obj, rscratch1)); + __ ldrb(rscratch2, Address(obj, rscratch1)); __ cbz(rscratch2, L_already_dirty); __ strb(zr, Address(obj, rscratch1)); __ bind(L_already_dirty); @@ -636,19 +633,21 @@ void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssemb return; } - ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set(); - CardTable* ct = bs->card_table(); - Label L_loop, L_done; const Register end = count; - __ cbz(count, L_done); // zero count - nothing to do + // Zero count? Nothing to do. + __ cbz(count, L_done); - __ lea(end, Address(start, count, Address::lsl(LogBytesPerHeapOop))); // end = start + count << LogBytesPerHeapOop - __ sub(end, end, BytesPerHeapOop); // last element address to make inclusive + // end = start + count << LogBytesPerHeapOop + // last element address to make inclusive + __ lea(end, Address(start, count, Address::lsl(LogBytesPerHeapOop))); + __ sub(end, end, BytesPerHeapOop); __ lsr(start, start, CardTable::card_shift()); __ lsr(end, end, CardTable::card_shift()); - __ sub(count, end, start); // number of bytes to copy + + // number of bytes to copy + __ sub(count, end, start); __ load_byte_map_base(scratch); __ add(start, start, scratch); diff --git a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp index e7b7ba40f381c..f55a4b91d2880 100644 --- a/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.hpp @@ -62,7 +62,9 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler { void resolve_forward_pointer_not_null(MacroAssembler* masm, Register dst, Register tmp = noreg); void load_reference_barrier(MacroAssembler* masm, Register dst, Address load_addr, DecoratorSet decorators); - void gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, Register start, Register count, Register scratch, RegSet saved_regs); + void gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, + Register start, Register count, + Register scratch, RegSet saved_regs); public: diff --git a/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp b/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp index 02f5fd40d5659..e2ccfdc991713 100644 --- a/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp +++ b/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp @@ -599,7 +599,7 @@ void ShenandoahBarrierSetAssembler::load_at( void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register base, RegisterOrConstant ind_or_offs, Register tmp) { if (!ShenandoahHeap::heap()->mode()->is_generational()) { - return; + return; } ShenandoahBarrierSet* ctbs = ShenandoahBarrierSet::barrier_set(); @@ -804,10 +804,13 @@ void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssemb CardTable* ct = bs->card_table(); assert_different_registers(addr, count, R0); - Label Lskip_loop, Lstore_loop; + Label L_skip_loop, L_store_loop; __ sldi_(count, count, LogBytesPerHeapOop); - __ beq(CCR0, Lskip_loop); // zero length + + // Zero length? Skip. + __ beq(CCR0, L_skip_loop); + __ addi(count, count, -BytesPerHeapOop); __ add(count, addr, count); // Use two shifts to clear out those low order two bits! (Cannot opt. into 1.) @@ -818,12 +821,13 @@ void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssemb __ addi(count, count, 1); __ li(R0, 0); __ mtctr(count); + // Byte store loop - __ bind(Lstore_loop); + __ bind(L_store_loop); __ stb(R0, 0, addr); __ addi(addr, addr, 1); - __ bdnz(Lstore_loop); - __ bind(Lskip_loop); + __ bdnz(L_store_loop); + __ bind(L_skip_loop); } #undef __ diff --git a/src/hotspot/cpu/x86/gc/shenandoah/c1/shenandoahBarrierSetC1_x86.cpp b/src/hotspot/cpu/x86/gc/shenandoah/c1/shenandoahBarrierSetC1_x86.cpp index de50bcd3799f4..78a2443bb526a 100644 --- a/src/hotspot/cpu/x86/gc/shenandoah/c1/shenandoahBarrierSetC1_x86.cpp +++ b/src/hotspot/cpu/x86/gc/shenandoah/c1/shenandoahBarrierSetC1_x86.cpp @@ -94,7 +94,7 @@ LIR_Opr ShenandoahBarrierSetC1::atomic_cmpxchg_at_resolved(LIRAccess& access, LI } } - LIR_Opr result = BarrierSetC1::atomic_cmpxchg_at_resolved(access, cmp_value, new_value); + LIR_Opr result = BarrierSetC1::atomic_cmpxchg_at_resolved(access, cmp_value, new_value); if (access.is_oop()) { post_barrier(access, access.resolved_addr(), new_value.result()); diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp index 9b5bccc158d79..ab3d4b645e187 100644 --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp @@ -127,9 +127,8 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0; bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops); - // We need to squirrel away the original element count because the - // array copy assembly will destroy the value and we need it for the - // card marking barrier. + // We need to save the original element count because the array copy stub + // will destroy the value and we need it for the card marking barrier. #ifdef _LP64 if (!checkcast) { if (!obj_int) { @@ -141,7 +140,7 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec } } #else -if (disjoint) { + if (disjoint) { __ mov(rdx, dst); // save 'to' } #endif @@ -166,10 +165,10 @@ if (disjoint) { #endif assert_different_registers(src, dst, count, thread); - Label done; + Label L_done; // Short-circuit if count == 0. __ testptr(count, count); - __ jcc(Assembler::zero, done); + __ jcc(Assembler::zero, L_done); // Avoid runtime call when not active. Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset())); @@ -180,7 +179,7 @@ if (disjoint) { flags = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::YOUNG_MARKING | ShenandoahHeap::OLD_MARKING; } __ testb(gc_state, flags); - __ jcc(Assembler::zero, done); + __ jcc(Assembler::zero, L_done); save_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ false); @@ -200,7 +199,7 @@ if (disjoint) { restore_machine_state(masm, /* handle_gpr = */ true, /* handle_fp = */ false); - __ bind(done); + __ bind(L_done); NOT_LP64(__ pop(thread);) } } @@ -214,7 +213,7 @@ void ShenandoahBarrierSetAssembler::arraycopy_epilogue(MacroAssembler* masm, Dec bool obj_int = type == T_OBJECT LP64_ONLY(&& UseCompressedOops); Register tmp = rax; -if (is_reference_type(type)) { + if (is_reference_type(type)) { #ifdef _LP64 if (!checkcast) { if (!obj_int) { @@ -663,8 +662,8 @@ void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register o // The calculation for byte_map_base is as follows: // byte_map_base = _byte_map - (uintptr_t(low_bound) >> card_shift); // So this essentially converts an address to a displacement and it will - // never need to be relocated. On 64bit however the value may be too - // large for a 32bit displacement. + // never need to be relocated. On 64-bit however the value may be too + // large for a 32-bit displacement. intptr_t byte_map_base = (intptr_t)ct->byte_map_base(); if (__ is_simm32(byte_map_base)) { card_addr = Address(noreg, obj, Address::times_1, byte_map_base); @@ -682,7 +681,7 @@ void ShenandoahBarrierSetAssembler::store_check(MacroAssembler* masm, Register o if (UseCondCardMark) { Label L_already_dirty; __ cmpb(card_addr, dirty); - __ jcc(Assembler::equal, L_already_dirty); + __ jccb(Assembler::equal, L_already_dirty); __ movb(card_addr, dirty); __ bind(L_already_dirty); } else { @@ -731,7 +730,7 @@ void ShenandoahBarrierSetAssembler::store_at(MacroAssembler* masm, DecoratorSet BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg); } else { iu_barrier(masm, val, tmp3); - // XXX: store_check missing from upstream + // TODO: store_check missing in upstream BarrierSetAssembler::store_at(masm, decorators, type, Address(tmp1, 0), val, noreg, noreg, noreg); store_check(masm, tmp1); } @@ -939,7 +938,9 @@ void ShenandoahBarrierSetAssembler::cmpxchg_oop(MacroAssembler* masm, #define TIMES_OOP (UseCompressedOops ? Address::times_4 : Address::times_8) -void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, Register addr, Register count, Register tmp) { +void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, + Register addr, Register count, + Register tmp) { if (!ShenandoahHeap::heap()->mode()->is_generational()) { return; } @@ -952,9 +953,9 @@ void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssemb const Register end = count; assert_different_registers(addr, end); + // Zero count? Nothing to do. __ testl(count, count); - __ jcc(Assembler::zero, L_done); // zero count - nothing to do - + __ jccb(Assembler::zero, L_done); #ifdef _LP64 __ leaq(end, Address(addr, count, TIMES_OOP, 0)); // end == addr+count*oop_size @@ -965,23 +966,25 @@ void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssemb __ mov64(tmp, disp); __ addptr(addr, tmp); -__ BIND(L_loop); + + __ BIND(L_loop); __ movb(Address(addr, count, Address::times_1), 0); __ decrement(count); - __ jcc(Assembler::greaterEqual, L_loop); + __ jccb(Assembler::greaterEqual, L_loop); #else - __ lea(end, Address(addr, count, Address::times_ptr, -wordSize)); + __ lea(end, Address(addr, count, Address::times_ptr, -wordSize)); __ shrptr(addr, CardTable::card_shift()); - __ shrptr(end, CardTable::card_shift()); + __ shrptr(end, CardTable::card_shift()); __ subptr(end, addr); // end --> count -__ BIND(L_loop); + + __ BIND(L_loop); Address cardtable(addr, count, Address::times_1, disp); __ movb(cardtable, 0); __ decrement(count); - __ jcc(Assembler::greaterEqual, L_loop); + __ jccb(Assembler::greaterEqual, L_loop); #endif -__ BIND(L_done); + __ BIND(L_done); } #undef __ diff --git a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp index 77da40a66d531..bea2174aafe3b 100644 --- a/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp +++ b/src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.hpp @@ -61,7 +61,9 @@ class ShenandoahBarrierSetAssembler: public BarrierSetAssembler { void store_check(MacroAssembler* masm, Register obj); - void gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, Register addr, Register count, Register tmp); + void gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators, + Register addr, Register count, + Register tmp); public: void iu_barrier(MacroAssembler* masm, Register dst, Register tmp); diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp index f72d292c88556..def68a375ddf7 100644 --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp @@ -476,17 +476,11 @@ void ShenandoahBarrierSetC2::post_barrier(GraphKit* kit, return; } - ShenandoahBarrierSet* ctbs = barrier_set_cast(BarrierSet::barrier_set()); - CardTable* ct = ctbs->card_table(); - // No store check needed if we're storing a nullptr or an old object - // (latter case is probably a string constant). The concurrent - // mark sweep garbage collector, however, needs to have all nonNull - // oop updates flagged via card-marks. + // No store check needed if we're storing a null. if (val != nullptr && val->is_Con()) { // must be either an oop or NULL const Type* t = val->bottom_type(); if (t == TypePtr::NULL_PTR || t == Type::TOP) - // stores of null never (?) need barriers return; } @@ -613,7 +607,8 @@ Node* ShenandoahBarrierSetC2::store_at_resolved(C2Access& access, C2AccessValue& bool anonymous = (decorators & ON_UNKNOWN_OOP_REF) != 0; bool is_array = (decorators & IS_ARRAY) != 0; bool use_precise = is_array || anonymous; - post_barrier(kit, kit->control(), access.raw_access(), access.base(), adr, adr_idx, val.node(), access.type(), use_precise); + post_barrier(kit, kit->control(), access.raw_access(), access.base(), + adr, adr_idx, val.node(), access.type(), use_precise); return result; } else { assert(access.is_opt_access(), "only for optimization passes"); @@ -739,7 +734,8 @@ Node* ShenandoahBarrierSetC2::atomic_cmpxchg_val_at_resolved(C2AtomicParseAccess } #endif load_store = kit->gvn().transform(new ShenandoahLoadReferenceBarrierNode(nullptr, load_store, access.decorators())); - post_barrier(kit, kit->control(), access.raw_access(), access.base(), access.addr().node(), access.alias_idx(), new_val, T_OBJECT, true); + post_barrier(kit, kit->control(), access.raw_access(), access.base(), + access.addr().node(), access.alias_idx(), new_val, T_OBJECT, true); return load_store; } return BarrierSetC2::atomic_cmpxchg_val_at_resolved(access, expected_val, new_val, value_type); @@ -902,7 +898,7 @@ bool ShenandoahBarrierSetC2::clone_needs_barrier(Node* src, PhaseGVN& gvn) { } } else { return true; - } + } } else if (src_type->isa_aryptr()) { BasicType src_elem = src_type->isa_aryptr()->elem()->array_element_basic_type(); if (is_reference_type(src_elem, true)) { @@ -1018,21 +1014,20 @@ void ShenandoahBarrierSetC2::eliminate_gc_barrier(PhaseMacroExpand* macro, Node* shenandoah_eliminate_wb_pre(node, ¯o->igvn()); } if (node->Opcode() == Op_CastP2X && ShenandoahHeap::heap()->mode()->is_generational()) { - assert(node->Opcode() == Op_CastP2X, "ConvP2XNode required"); - Node *shift = node->unique_out(); - Node *addp = shift->unique_out(); - for (DUIterator_Last jmin, j = addp->last_outs(jmin); j >= jmin; --j) { - Node *mem = addp->last_out(j); - if (UseCondCardMark && mem->is_Load()) { - assert(mem->Opcode() == Op_LoadB, "unexpected code shape"); - // The load is checking if the card has been written so - // replace it with zero to fold the test. - macro->replace_node(mem, macro->intcon(0)); - continue; - } - assert(mem->is_Store(), "store required"); - macro->replace_node(mem, mem->in(MemNode::Memory)); - } + Node* shift = node->unique_out(); + Node* addp = shift->unique_out(); + for (DUIterator_Last jmin, j = addp->last_outs(jmin); j >= jmin; --j) { + Node* mem = addp->last_out(j); + if (UseCondCardMark && mem->is_Load()) { + assert(mem->Opcode() == Op_LoadB, "unexpected code shape"); + // The load is checking if the card has been written so + // replace it with zero to fold the test. + macro->replace_node(mem, macro->intcon(0)); + continue; + } + assert(mem->is_Store(), "store required"); + macro->replace_node(mem, mem->in(MemNode::Memory)); + } } } diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp index e6d3a342aa11e..b37bedbcb146a 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp @@ -56,6 +56,7 @@ const double ShenandoahAdaptiveHeuristics::HIGHEST_EXPECTED_AVAILABLE_AT_END = 0 const double ShenandoahAdaptiveHeuristics::MINIMUM_CONFIDENCE = 0.319; // 25% const double ShenandoahAdaptiveHeuristics::MAXIMUM_CONFIDENCE = 3.291; // 99.9% +// TODO: Provide comment here or remove if not used const uint ShenandoahAdaptiveHeuristics::MINIMUM_RESIZE_INTERVAL = 10; ShenandoahAdaptiveHeuristics::ShenandoahAdaptiveHeuristics(ShenandoahGeneration* generation) : @@ -95,6 +96,7 @@ void ShenandoahAdaptiveHeuristics::choose_collection_set_from_regiondata(Shenand // particular, regions that have reached tenure age will be sorted into this array before younger regions that contain // more garbage. This represents one of the reasons why we keep looking at regions even after we decide, for example, // to exclude one of the regions because it might require evacuation of too much live data. + // TODO: Split it in the separate methods for clarity. bool is_generational = heap->mode()->is_generational(); bool is_global = _generation->is_global(); size_t capacity = heap->young_generation()->max_capacity(); @@ -261,9 +263,10 @@ void ShenandoahAdaptiveHeuristics::record_success_concurrent(bool abbreviated) { z_score = (double(available) - available_avg) / available_sd; log_debug(gc, ergo)("%s Available: " SIZE_FORMAT " %sB, z-score=%.3f. Average available: %.1f %sB +/- %.1f %sB.", _generation->name(), - byte_size_in_proper_unit(available), proper_unit_for_byte_size(available), z_score, + byte_size_in_proper_unit(available), proper_unit_for_byte_size(available), + z_score, byte_size_in_proper_unit(available_avg), proper_unit_for_byte_size(available_avg), - byte_size_in_proper_unit(available_sd), proper_unit_for_byte_size(available_sd)); + byte_size_in_proper_unit(available_sd), proper_unit_for_byte_size(available_sd)); } _available.add(double(available)); @@ -336,7 +339,7 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { size_t usable = ShenandoahHeap::heap()->free_set()->available(); if (usable < available) { log_debug(gc)("Usable (" SIZE_FORMAT "%s) is less than available (" SIZE_FORMAT "%s)", - byte_size_in_proper_unit(usable), proper_unit_for_byte_size(usable), + byte_size_in_proper_unit(usable), proper_unit_for_byte_size(usable), byte_size_in_proper_unit(available), proper_unit_for_byte_size(available)); available = usable; } @@ -350,8 +353,8 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { if (available < min_threshold) { log_info(gc)("Trigger (%s): Free (" SIZE_FORMAT "%s) is below minimum threshold (" SIZE_FORMAT "%s)", _generation->name(), - byte_size_in_proper_unit(available), proper_unit_for_byte_size(available), - byte_size_in_proper_unit(min_threshold), proper_unit_for_byte_size(min_threshold)); + byte_size_in_proper_unit(available), proper_unit_for_byte_size(available), + byte_size_in_proper_unit(min_threshold), proper_unit_for_byte_size(min_threshold)); return resize_and_evaluate(); } @@ -362,8 +365,8 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { if (available < init_threshold) { log_info(gc)("Trigger (%s): Learning " SIZE_FORMAT " of " SIZE_FORMAT ". Free (" SIZE_FORMAT "%s) is below initial threshold (" SIZE_FORMAT "%s)", _generation->name(), _gc_times_learned + 1, max_learn, - byte_size_in_proper_unit(available), proper_unit_for_byte_size(available), - byte_size_in_proper_unit(init_threshold), proper_unit_for_byte_size(init_threshold)); + byte_size_in_proper_unit(available), proper_unit_for_byte_size(available), + byte_size_in_proper_unit(init_threshold), proper_unit_for_byte_size(init_threshold)); return true; } } @@ -421,13 +424,13 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { double avg_alloc_rate = _allocation_rate.upper_bound(_margin_of_error_sd); log_debug(gc)("%s: average GC time: %.2f ms, allocation rate: %.0f %s/s", - _generation->name(), avg_cycle_time * 1000, byte_size_in_proper_unit(avg_alloc_rate), proper_unit_for_byte_size(avg_alloc_rate)); + _generation->name(), avg_cycle_time * 1000, + byte_size_in_proper_unit(avg_alloc_rate), proper_unit_for_byte_size(avg_alloc_rate)); if (avg_cycle_time > allocation_headroom / avg_alloc_rate) { - log_info(gc)("Trigger (%s): Average GC time (%.2f ms) is above the time for average allocation rate (%.0f %sB/s) to deplete free headroom (" SIZE_FORMAT "%s) (margin of error = %.2f)", _generation->name(), avg_cycle_time * 1000, - byte_size_in_proper_unit(avg_alloc_rate), proper_unit_for_byte_size(avg_alloc_rate), + byte_size_in_proper_unit(avg_alloc_rate), proper_unit_for_byte_size(avg_alloc_rate), byte_size_in_proper_unit(allocation_headroom), proper_unit_for_byte_size(allocation_headroom), _margin_of_error_sd); @@ -447,7 +450,6 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { _generation->name(), avg_cycle_time * 1000, byte_size_in_proper_unit(rate), proper_unit_for_byte_size(rate), byte_size_in_proper_unit(allocation_headroom), proper_unit_for_byte_size(allocation_headroom), - _spike_threshold_sd); _last_trigger = SPIKE; return resize_and_evaluate(); @@ -464,14 +466,14 @@ bool ShenandoahAdaptiveHeuristics::resize_and_evaluate() { } if (_cycles_since_last_resize <= MINIMUM_RESIZE_INTERVAL) { - log_info(gc, ergo)("Not resizing %s for another " UINT32_FORMAT " cycles.", - _generation->name(), _cycles_since_last_resize); + log_info(gc, ergo)("Not resizing %s for another " UINT32_FORMAT " cycles", + _generation->name(), _cycles_since_last_resize); return true; } if (!heap->generation_sizer()->transfer_capacity(_generation)) { // We could not enlarge our generation, so we must start a gc cycle. - log_info(gc, ergo)("Could not increase size of %s, begin gc cycle.", _generation->name()); + log_info(gc, ergo)("Could not increase size of %s, begin gc cycle", _generation->name()); return true; } diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp index d2f40589b7abb..c7096d33e7a7b 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp @@ -32,7 +32,12 @@ #include "logging/logTag.hpp" #include "runtime/os.hpp" -ShenandoahAggressiveHeuristics::ShenandoahAggressiveHeuristics(ShenandoahGeneration* generation) : ShenandoahHeuristics(generation) { +ShenandoahAggressiveHeuristics::ShenandoahAggressiveHeuristics(ShenandoahGeneration* generation) : + ShenandoahHeuristics(generation) { + + assert(!ShenandoahHeap::heap()->mode()->is_generational(), + "Aggressive heuristics is not available in generational mode"); + // Do not shortcut evacuation SHENANDOAH_ERGO_OVERRIDE_DEFAULT(ShenandoahImmediateThreshold, 100); @@ -49,10 +54,8 @@ ShenandoahAggressiveHeuristics::ShenandoahAggressiveHeuristics(ShenandoahGenerat void ShenandoahAggressiveHeuristics::choose_collection_set_from_regiondata(ShenandoahCollectionSet* cset, RegionData* data, size_t size, size_t free) { - assert(!ShenandoahHeap::heap()->mode()->is_generational(), "AggressiveHeuristics not appropriate in generational mode"); - - // Note that there's no bound on collection set size. If we try to collect too much memory, we'll get an alloc - // failure during collection and we'll degenerate. + // Note that there is no bound on collection set size. If we try to collect too much memory, + // we'll get an allocation failure during collection and slide to degenerated GC. for (size_t idx = 0; idx < size; idx++) { ShenandoahHeapRegion* r = data[idx]._region; if (r->garbage() > 0) { diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp index d96f7ce0a83ab..a84c23b678cd5 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp @@ -48,8 +48,8 @@ ShenandoahCompactHeuristics::ShenandoahCompactHeuristics(ShenandoahGeneration* g bool ShenandoahCompactHeuristics::should_start_gc() { size_t max_capacity = _generation->max_capacity(); - size_t capacity = _generation->soft_max_capacity(); - size_t available = _generation->available(); + size_t capacity = _generation->soft_max_capacity(); + size_t available = _generation->available(); // Make sure the code below treats available without the soft tail. size_t soft_tail = max_capacity - capacity; diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp index 3690458f6f190..2a811b591262e 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp @@ -76,21 +76,24 @@ ShenandoahHeuristics::~ShenandoahHeuristics() { FREE_C_HEAP_ARRAY(RegionGarbage, _region_data); } -size_t ShenandoahHeuristics::select_aged_regions(size_t old_available, size_t num_regions, bool preselected_regions[]) { +size_t ShenandoahHeuristics::select_aged_regions(size_t old_available, size_t num_regions, bool* preselected_regions) { ShenandoahHeap* heap = ShenandoahHeap::heap(); + if (!heap->mode()->is_generational()) { + // TODO: Do we need this check, or assert is enough? + return 0; + } + size_t old_consumed = 0; - if (heap->mode()->is_generational()) { - for (size_t i = 0; i < num_regions; i++) { - ShenandoahHeapRegion* region = heap->get_region(i); - if (in_generation(region) && !region->is_empty() && region->is_regular() && (region->age() >= InitialTenuringThreshold)) { - size_t promotion_need = (size_t) (region->get_live_data_bytes() * ShenandoahEvacWaste); - if (old_consumed + promotion_need < old_available) { - old_consumed += promotion_need; - preselected_regions[i] = true; - } - // Note that we keep going even if one region is excluded from selection. Subsequent regions may be selected - // if they have smaller live data. + for (size_t i = 0; i < num_regions; i++) { + ShenandoahHeapRegion* region = heap->get_region(i); + if (in_generation(region) && !region->is_empty() && region->is_regular() && (region->age() >= InitialTenuringThreshold)) { + size_t promotion_need = (size_t) (region->get_live_data_bytes() * ShenandoahEvacWaste); + if (old_consumed + promotion_need < old_available) { + old_consumed += promotion_need; + preselected_regions[i] = true; } + // Note that we keep going even if one region is excluded from selection. + // Subsequent regions may be selected if they have smaller live data. } } return old_consumed; @@ -101,7 +104,7 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec bool is_generational = heap->mode()->is_generational(); assert(collection_set->count() == 0, "Must be empty"); - assert(!_generation->is_old(), "Old GC invokes ShenandoahOldHeuristics::choose_collection_set()"); + assert(!is_generational || !_generation->is_old(), "Old GC invokes ShenandoahOldHeuristics::choose_collection_set()"); // Check all pinned regions have updated status before choosing the collection set. heap->assert_pinned_region_status(); @@ -121,7 +124,6 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec size_t free = 0; size_t free_regions = 0; - size_t live_memory = 0; for (size_t i = 0; i < num_regions; i++) { ShenandoahHeapRegion* region = heap->get_region(i); @@ -142,7 +144,6 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec region->make_trash_immediate(); } else { assert(!_generation->is_old(), "OLD is handled elsewhere"); - live_memory += region->get_live_data_bytes(); // This is our candidate for later consideration. candidates[cand_idx]._region = region; if (is_generational && collection_set->is_preselected(i)) { @@ -168,15 +169,11 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec // Count only the start. Continuations would be counted on "trash" path immediate_regions++; immediate_garbage += garbage; - } else { - live_memory += region->get_live_data_bytes(); } } else if (region->is_trash()) { // Count in just trashed collection set, during coalesced CM-with-UR immediate_regions++; immediate_garbage += garbage; - } else { // region->is_humongous_cont() and !region->is_trash() - live_memory += region->get_live_data_bytes(); } } @@ -194,14 +191,15 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec if (immediate_percent <= ShenandoahImmediateThreshold) { if (old_heuristics != nullptr) { old_heuristics->prime_collection_set(collection_set); + } else { + // This is a global collection and does not need to prime cset } - // else, this is global collection and doesn't need to prime_collection_set - // Add young-gen regions into the collection set. This is a virtual call, implemented differently by each - // of the heuristics subclasses. + // Call the subclasses to add young-gen regions into the collection set. choose_collection_set_from_regiondata(collection_set, candidates, cand_idx, immediate_garbage + free); } else { - // we're going to skip evacuation and update refs because we reclaimed sufficient amounts of immediate garbage. + // We are going to skip evacuation and update refs because we reclaimed + // sufficient amounts of immediate garbage. heap->shenandoah_policy()->record_abbreviated_cycle(); } @@ -214,8 +212,8 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec size_t collectable_garbage_percent = (total_garbage == 0) ? 0 : (collectable_garbage * 100 / total_garbage); log_info(gc, ergo)("Collectable Garbage: " SIZE_FORMAT "%s (" SIZE_FORMAT "%%), " - "Immediate: " SIZE_FORMAT "%s (" SIZE_FORMAT "%%) R: " SIZE_FORMAT ", " - "CSet: " SIZE_FORMAT "%s (" SIZE_FORMAT "%%) R: " SIZE_FORMAT, + "Immediate: " SIZE_FORMAT "%s (" SIZE_FORMAT "%%), " SIZE_FORMAT " regions, " + "CSet: " SIZE_FORMAT "%s (" SIZE_FORMAT "%%), " SIZE_FORMAT " regions", byte_size_in_proper_unit(collectable_garbage), proper_unit_for_byte_size(collectable_garbage), @@ -232,18 +230,18 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec collection_set->count()); if (collection_set->garbage() > 0) { - size_t young_evac_bytes = collection_set->get_young_bytes_reserved_for_evacuation(); + size_t young_evac_bytes = collection_set->get_young_bytes_reserved_for_evacuation(); size_t promote_evac_bytes = collection_set->get_young_bytes_to_be_promoted(); - size_t old_evac_bytes = collection_set->get_old_bytes_reserved_for_evacuation(); - size_t total_evac_bytes = young_evac_bytes + promote_evac_bytes + old_evac_bytes; + size_t old_evac_bytes = collection_set->get_old_bytes_reserved_for_evacuation(); + size_t total_evac_bytes = young_evac_bytes + promote_evac_bytes + old_evac_bytes; log_info(gc, ergo)("Evacuation Targets: YOUNG: " SIZE_FORMAT "%s, " "PROMOTE: " SIZE_FORMAT "%s, " "OLD: " SIZE_FORMAT "%s, " "TOTAL: " SIZE_FORMAT "%s", - byte_size_in_proper_unit(young_evac_bytes), proper_unit_for_byte_size(young_evac_bytes), + byte_size_in_proper_unit(young_evac_bytes), proper_unit_for_byte_size(young_evac_bytes), byte_size_in_proper_unit(promote_evac_bytes), proper_unit_for_byte_size(promote_evac_bytes), - byte_size_in_proper_unit(old_evac_bytes), proper_unit_for_byte_size(old_evac_bytes), - byte_size_in_proper_unit(total_evac_bytes), proper_unit_for_byte_size(total_evac_bytes)); + byte_size_in_proper_unit(old_evac_bytes), proper_unit_for_byte_size(old_evac_bytes), + byte_size_in_proper_unit(total_evac_bytes), proper_unit_for_byte_size(total_evac_bytes)); } } diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp index ffedff5e84625..aba842c24552d 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp @@ -75,18 +75,17 @@ class ShenandoahHeuristics : public CHeapObj { ShenandoahGeneration* _generation; - // if (_generation->type() == GLOBAL) _region_data represents - // the results of most recently completed global marking pass - // if (_generation->type() == OLD) _region_data represents - // the results of most recently completed old-gen marking pass - // if (_generation->type() == YOUNG) _region_data represents - // the results of most recently completed young-gen marking pass + // Depending on generation mode, region data represents the results of the relevant + // most recently completed marking pass: + // - in GLOBAL mode, global marking pass + // - in OLD mode, old-gen marking pass + // - in YOUNG mode, young-gen marking pass // - // Note that there is some redundancy represented in _region_data because - // each instance is an array large enough to hold all regions. However, - // any region in young-gen is not in old-gen. And any time we are + // Note that there is some redundancy represented in region data because + // each instance is an array large enough to hold all regions. However, + // any region in young-gen is not in old-gen. And any time we are // making use of the GLOBAL data, there is no need to maintain the - // YOUNG or OLD data. Consider this redundancy of data structure to + // YOUNG or OLD data. Consider this redundancy of data structure to // have negligible cost unless proven otherwise. RegionData* _region_data; @@ -157,7 +156,7 @@ class ShenandoahHeuristics : public CHeapObj { virtual void reset_gc_learning(); - virtual size_t select_aged_regions(size_t old_available, size_t num_regions, bool preselected_regions[]); + virtual size_t select_aged_regions(size_t old_available, size_t num_regions, bool* preselected_regions); virtual void choose_collection_set(ShenandoahCollectionSet* collection_set, ShenandoahOldHeuristics* old_heuristics); diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp index 756aeaa0e8f5b..4ce60002f376b 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp @@ -83,15 +83,16 @@ bool ShenandoahOldHeuristics::prime_collection_set(ShenandoahCollectionSet* coll // If we choose region r to be collected, then we need to decrease the capacity to hold other evacuations by // the size of r's free memory. - // It's probably overkill to compensate with lost_evacuation_capacity. But it's the safe thing to do and - // has minimal impact on content of primed collection set. - if (r->get_live_data_bytes() + lost_evacuation_capacity <= remaining_old_evacuation_budget) { + // It's probably overkill to compensate with lost_evacuation_capacity. + // But it's the safe thing to do and has minimal impact on content of primed collection set. + size_t live = r->get_live_data_bytes(); + if (live + lost_evacuation_capacity <= remaining_old_evacuation_budget) { // Decrement remaining evacuation budget by bytes that will be copied. lost_evacuation_capacity += r->free(); - remaining_old_evacuation_budget -= r->get_live_data_bytes(); + remaining_old_evacuation_budget -= live; collection_set->add_region(r); included_old_regions++; - evacuated_old_bytes += r->get_live_data_bytes(); + evacuated_old_bytes += live; collected_old_bytes += r->garbage(); consume_old_collection_candidate(); } else { @@ -121,7 +122,7 @@ bool ShenandoahOldHeuristics::prime_collection_set(ShenandoahCollectionSet* coll // if they are all pinned we transition to a state that will allow us to make these uncollected // (pinned) regions parseable. if (all_candidates_are_pinned()) { - log_info(gc)("All candidate regions " UINT32_FORMAT " are pinned.", unprocessed_old_collection_candidates()); + log_info(gc)("All candidate regions " UINT32_FORMAT " are pinned", unprocessed_old_collection_candidates()); _old_generation->transition_to(ShenandoahOldGeneration::WAITING_FOR_FILL); } } @@ -137,7 +138,7 @@ bool ShenandoahOldHeuristics::all_candidates_are_pinned() { #endif for (uint i = _next_old_collection_candidate; i < _last_old_collection_candidate; ++i) { - auto region = _region_data[i]._region; + ShenandoahHeapRegion* region = _region_data[i]._region; if (!region->is_pinned()) { return false; } @@ -185,7 +186,7 @@ void ShenandoahOldHeuristics::slide_pinned_regions_to_front() { // | | next region for mixed collections // | Write pointer is here. We know this region is already in the cset // | so we can clobber it with the next pinned region we find. - for (int32_t search = write_index - 1; search >= (int32_t)_first_pinned_candidate; --search) { + for (int32_t search = (int32_t)write_index - 1; search >= (int32_t)_first_pinned_candidate; --search) { RegionData& skipped = _region_data[search]; if (skipped._region->is_pinned()) { RegionData& available_slot = _region_data[write_index]; @@ -204,8 +205,8 @@ void ShenandoahOldHeuristics::slide_pinned_regions_to_front() { // Both arguments are don't cares for old-gen collections void ShenandoahOldHeuristics::choose_collection_set(ShenandoahCollectionSet* collection_set, ShenandoahOldHeuristics* old_heuristics) { - assert((collection_set == nullptr) && (old_heuristics == nullptr), - "Expect null arguments in ShenandoahOldHeuristics::choose_collection_set()"); + assert(collection_set == nullptr, "Expect null"); + assert(old_heuristics == nullptr, "Expect null"); // Old-gen doesn't actually choose a collection set to be evacuated by its own gang of worker tasks. // Instead, it computes the set of regions to be evacuated by subsequent young-gen evacuation passes. prepare_for_old_collections(); @@ -295,7 +296,7 @@ void ShenandoahOldHeuristics::prepare_for_old_collections() { log_info(gc)("Old-Gen Collectable Garbage: " SIZE_FORMAT "%s over " UINT32_FORMAT " regions, " "Old-Gen Immediate Garbage: " SIZE_FORMAT "%s over " SIZE_FORMAT " regions.", byte_size_in_proper_unit(collectable_garbage), proper_unit_for_byte_size(collectable_garbage), _last_old_collection_candidate, - byte_size_in_proper_unit(immediate_garbage), proper_unit_for_byte_size(immediate_garbage), immediate_regions); + byte_size_in_proper_unit(immediate_garbage), proper_unit_for_byte_size(immediate_garbage), immediate_regions); if (unprocessed_old_collection_candidates() == 0) { _old_generation->transition_to(ShenandoahOldGeneration::IDLE); @@ -304,6 +305,7 @@ void ShenandoahOldHeuristics::prepare_for_old_collections() { } } +// TODO: Unused? uint ShenandoahOldHeuristics::last_old_collection_candidate_index() { return _last_old_collection_candidate; } diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp index 1fa6ec034fcbc..847bd283df22a 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp @@ -55,7 +55,7 @@ void ShenandoahPassiveHeuristics::choose_collection_set_from_regiondata(Shenando // Do not select too large CSet that would overflow the available free space. // Take at least the entire evacuation reserve, and be free to overflow to free space. size_t max_capacity = ShenandoahHeap::heap()->max_capacity(); - size_t available = MAX2((max_capacity / 100) * ShenandoahEvacReserve, actual_free); + size_t available = MAX2(max_capacity / 100 * ShenandoahEvacReserve, actual_free); size_t max_cset = (size_t)(available / ShenandoahEvacWaste); log_info(gc, ergo)("CSet Selection. Actual Free: " SIZE_FORMAT "%s, Max CSet: " SIZE_FORMAT "%s", diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp index 779677e221354..ce82680a17068 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp @@ -43,8 +43,8 @@ ShenandoahStaticHeuristics::~ShenandoahStaticHeuristics() {} bool ShenandoahStaticHeuristics::should_start_gc() { size_t max_capacity = _generation->max_capacity(); - size_t capacity = _generation->soft_max_capacity(); - size_t available = _generation->available(); + size_t capacity = _generation->soft_max_capacity(); + size_t available = _generation->available(); // Make sure the code below treats available without the soft tail. size_t soft_tail = max_capacity - capacity; diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp index 4b7e0a3193880..9c5bce4abe797 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp @@ -63,3 +63,4 @@ ShenandoahHeuristics* ShenandoahGenerationalMode::initialize_heuristics(Shenando return new ShenandoahAdaptiveHeuristics(generation); } + diff --git a/src/hotspot/share/gc/shenandoah/shenandoahAsserts.cpp b/src/hotspot/share/gc/shenandoah/shenandoahAsserts.cpp index 45ff4e1ca0b71..918f79d6bd273 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahAsserts.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahAsserts.cpp @@ -429,7 +429,8 @@ void ShenandoahAsserts::assert_heaplocked_or_safepoint(const char* file, int lin report_vm_error(file, line, msg.buffer()); } -// unlike assert_heaplocked_or_safepoint(), this does not require current thread in safepoint to be a VM-thread +// Unlike assert_heaplocked_or_safepoint(), this does not require current thread in safepoint to be a VM thread +// TODO: This should be more aptly named. Nothing in this method checks we are actually in Full GC. void ShenandoahAsserts::assert_heaplocked_or_fullgc_safepoint(const char* file, int line) { ShenandoahHeap* heap = ShenandoahHeap::heap(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp index ea74454411e35..49d08c78cbc7d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp @@ -133,6 +133,7 @@ void ShenandoahBarrierSet::on_thread_detach(Thread *thread) { // CAUTION: retire_plab may register the remnant filler object with the remembered set scanner without a lock. // This is safe iff it is assured that each PLAB is a whole-number multiple of card-mark memory size and each // PLAB is aligned with the start of each card's memory range. + // TODO: Assert this in retire_plab? if (plab != nullptr) { _heap->retire_plab(plab); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp index f039120a92c31..4927b599711cc 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp @@ -49,7 +49,9 @@ class ShenandoahBarrierSet: public BarrierSet { return barrier_set_cast(BarrierSet::barrier_set()); } - inline ShenandoahCardTable* card_table() { return _card_table; } + inline ShenandoahCardTable* card_table() { + return _card_table; + } static ShenandoahSATBMarkQueueSet& satb_mark_queue_set() { return barrier_set()->_satb_mark_queue_set; @@ -117,7 +119,7 @@ class ShenandoahBarrierSet: public BarrierSet { inline oop oop_xchg(DecoratorSet decorators, T* addr, oop new_value); template - void write_ref_field_post(T* field, oop newVal); + void write_ref_field_post(T* field); void write_ref_array(HeapWord* start, size_t count); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp index db17c6bcf4fc6..e64a6983d0dae 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp @@ -186,7 +186,7 @@ inline void ShenandoahBarrierSet::keep_alive_if_weak(DecoratorSet decorators, oo } template -inline void ShenandoahBarrierSet::write_ref_field_post(T* field, oop newVal) { +inline void ShenandoahBarrierSet::write_ref_field_post(T* field) { if (ShenandoahHeap::heap()->mode()->is_generational()) { volatile CardTable::CardValue* byte = card_table()->byte_for(field); *byte = CardTable::dirty_card_val(); @@ -256,7 +256,9 @@ inline oop ShenandoahBarrierSet::AccessBarrier::oop_loa template template inline void ShenandoahBarrierSet::AccessBarrier::oop_store_common(T* addr, oop value) { - shenandoah_assert_marked_if(nullptr, value, !CompressedOops::is_null(value) && ShenandoahHeap::heap()->is_evacuation_in_progress() && + shenandoah_assert_marked_if(nullptr, value, + !CompressedOops::is_null(value) && + ShenandoahHeap::heap()->is_evacuation_in_progress() && !(ShenandoahHeap::heap()->is_gc_generation_young() && ShenandoahHeap::heap()->heap_region_containing(value)->is_old())); shenandoah_assert_not_in_cset_if(addr, value, value != nullptr && !ShenandoahHeap::heap()->cancelled_gc()); ShenandoahBarrierSet* const bs = ShenandoahBarrierSet::barrier_set(); @@ -278,7 +280,8 @@ inline void ShenandoahBarrierSet::AccessBarrier::oop_st shenandoah_assert_not_forwarded_except (addr, value, value == nullptr || ShenandoahHeap::heap()->cancelled_gc() || !ShenandoahHeap::heap()->is_concurrent_mark_in_progress()); oop_store_common(addr, value); - ShenandoahBarrierSet::barrier_set()->write_ref_field_post(addr, value); + ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set(); + bs->write_ref_field_post(addr); } template @@ -300,7 +303,7 @@ inline oop ShenandoahBarrierSet::AccessBarrier::oop_ato assert((decorators & (AS_NO_KEEPALIVE | ON_UNKNOWN_OOP_REF)) == 0, "must be absent"); ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set(); oop result = bs->oop_cmpxchg(decorators, addr, compare_value, new_value); - bs->write_ref_field_post(addr, new_value); + bs->write_ref_field_post(addr); return result; } @@ -311,7 +314,7 @@ inline oop ShenandoahBarrierSet::AccessBarrier::oop_ato DecoratorSet resolved_decorators = AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength(base, offset); auto addr = AccessInternal::oop_field_addr(base, offset); oop result = bs->oop_cmpxchg(resolved_decorators, addr, compare_value, new_value); - bs->write_ref_field_post(addr, new_value); + bs->write_ref_field_post(addr); return result; } @@ -329,7 +332,7 @@ inline oop ShenandoahBarrierSet::AccessBarrier::oop_ato assert((decorators & (AS_NO_KEEPALIVE | ON_UNKNOWN_OOP_REF)) == 0, "must be absent"); ShenandoahBarrierSet* bs = ShenandoahBarrierSet::barrier_set(); oop result = bs->oop_xchg(decorators, addr, new_value); - bs->write_ref_field_post(addr, new_value); + bs->write_ref_field_post(addr); return result; } @@ -340,7 +343,7 @@ inline oop ShenandoahBarrierSet::AccessBarrier::oop_ato DecoratorSet resolved_decorators = AccessBarrierSupport::resolve_possibly_unknown_oop_ref_strength(base, offset); auto addr = AccessInternal::oop_field_addr(base, offset); oop result = bs->oop_xchg(resolved_decorators, addr, new_value); - bs->write_ref_field_post(addr, new_value); + bs->write_ref_field_post(addr); return result; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCardStats.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCardStats.cpp index abd34c2cbf04a..ef2d6e134b2e2 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCardStats.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCardStats.cpp @@ -34,7 +34,8 @@ void ShenandoahCardStats::log() const { log_info(gc,remset)("Card stats: dirty " SIZE_FORMAT " (max run: " SIZE_FORMAT ")," " clean " SIZE_FORMAT " (max run: " SIZE_FORMAT ")," " dirty scans/objs " SIZE_FORMAT, - _dirty_card_cnt, _max_dirty_run, _clean_card_cnt, _max_clean_run, + _dirty_card_cnt, _max_dirty_run, + _clean_card_cnt, _max_clean_run, _dirty_scan_obj_cnt); } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCardStats.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCardStats.hpp index 548b2e0174561..de21e226acbc1 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCardStats.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCardStats.hpp @@ -29,21 +29,21 @@ #include "gc/shenandoah/shenandoahNumberSeq.hpp" enum CardStatType { - DIRTY_RUN = 0, - CLEAN_RUN = 1, - DIRTY_CARDS = 2, - CLEAN_CARDS = 3, - MAX_DIRTY_RUN = 4, - MAX_CLEAN_RUN = 5, - DIRTY_SCAN_OBJS = 6, - ALTERNATIONS = 7, - MAX_CARD_STAT_TYPE = 8 + DIRTY_RUN = 0, + CLEAN_RUN = 1, + DIRTY_CARDS = 2, + CLEAN_CARDS = 3, + MAX_DIRTY_RUN = 4, + MAX_CLEAN_RUN = 5, + DIRTY_SCAN_OBJS = 6, + ALTERNATIONS = 7, + MAX_CARD_STAT_TYPE = 8 }; enum CardStatLogType { - CARD_STAT_SCAN_RS = 0, - CARD_STAT_UPDATE_REFS = 1, - MAX_CARD_STAT_LOG_TYPE = 2 + CARD_STAT_SCAN_RS = 0, + CARD_STAT_UPDATE_REFS = 1, + MAX_CARD_STAT_LOG_TYPE = 2 }; class ShenandoahCardStats: public CHeapObj { @@ -54,9 +54,6 @@ class ShenandoahCardStats: public CHeapObj { size_t _dirty_card_cnt; size_t _clean_card_cnt; - size_t _dirty_run; - size_t _clean_run; - size_t _max_dirty_run; size_t _max_clean_run; @@ -83,12 +80,12 @@ class ShenandoahCardStats: public CHeapObj { void record() { if (ShenandoahEnableCardStats) { // Update global stats for distribution of dirty/clean cards as a percentage of chunk - _local_card_stats[DIRTY_CARDS].add((double)_dirty_card_cnt*100/(double)_cards_in_cluster); - _local_card_stats[CLEAN_CARDS].add((double)_clean_card_cnt*100/(double)_cards_in_cluster); + _local_card_stats[DIRTY_CARDS].add(percent_of(_dirty_card_cnt, _cards_in_cluster)); + _local_card_stats[CLEAN_CARDS].add(percent_of(_clean_card_cnt, _cards_in_cluster)); // Update global stats for max dirty/clean run distribution as a percentage of chunk - _local_card_stats[MAX_DIRTY_RUN].add((double)_max_dirty_run*100/(double)_cards_in_cluster); - _local_card_stats[MAX_CLEAN_RUN].add((double)_max_clean_run*100/(double)_cards_in_cluster); + _local_card_stats[MAX_DIRTY_RUN].add(percent_of(_max_dirty_run, _cards_in_cluster)); + _local_card_stats[MAX_CLEAN_RUN].add(percent_of(_max_clean_run, _cards_in_cluster)); // Update global stats for dirty obj scan counts _local_card_stats[DIRTY_SCAN_OBJS].add(_dirty_scan_obj_cnt); @@ -107,7 +104,7 @@ class ShenandoahCardStats: public CHeapObj { } _dirty_card_cnt += len; assert(len <= _cards_in_cluster, "Error"); - _local_card_stats[DIRTY_RUN].add((double)len*100.0/(double)_cards_in_cluster); + _local_card_stats[DIRTY_RUN].add(percent_of(len, _cards_in_cluster)); } } @@ -119,7 +116,7 @@ class ShenandoahCardStats: public CHeapObj { } _clean_card_cnt += len; assert(len <= _cards_in_cluster, "Error"); - _local_card_stats[CLEAN_RUN].add((double)len*100.0/(double)_cards_in_cluster); + _local_card_stats[CLEAN_RUN].add(percent_of(len, _cards_in_cluster)); } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCardTable.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCardTable.cpp index 89714d266ac95..c2b17b4e654ea 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCardTable.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCardTable.cpp @@ -31,6 +31,9 @@ void ShenandoahCardTable::initialize() { CardTable::initialize(); _write_byte_map = _byte_map; _write_byte_map_base = _byte_map_base; + + // TODO: Why rs_align is 0 on page_size == os::vm_page_size? + // ReservedSpace constructor would assert rs_align >= os::vm_page_size(). const size_t rs_align = _page_size == (size_t) os::vm_page_size() ? 0 : MAX2(_page_size, (size_t) os::vm_allocation_granularity()); @@ -44,7 +47,7 @@ void ShenandoahCardTable::initialize() { _read_byte_map = (CardValue*) heap_rs.base(); _read_byte_map_base = _read_byte_map - (uintptr_t(low_bound) >> card_shift()); - log_trace(gc, barrier)("ShenandoahCardTable::ShenandoahCardTable: "); + log_trace(gc, barrier)("ShenandoahCardTable::ShenandoahCardTable:"); log_trace(gc, barrier)(" &_read_byte_map[0]: " INTPTR_FORMAT " &_read_byte_map[_last_valid_index]: " INTPTR_FORMAT, p2i(&_read_byte_map[0]), p2i(&_read_byte_map[last_valid_index()])); log_trace(gc, barrier)(" _read_byte_map_base: " INTPTR_FORMAT, p2i(_read_byte_map_base)); @@ -68,6 +71,7 @@ bool ShenandoahCardTable::is_in_young(const void* obj) const { return ShenandoahHeap::heap()->is_in_young(obj); } +// TODO: Seems to be unused bool ShenandoahCardTable::is_dirty(MemRegion mr) { for (size_t i = index_for(mr.start()); i <= index_for(mr.end() - 1); i++) { CardValue* byte = byte_for_index(i); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp index 67222fe82d4dc..c797ad21a7dd3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp @@ -91,23 +91,26 @@ void ShenandoahCollectionSet::add_region(ShenandoahHeapRegion* r) { _cset_map[r->index()] = 1; + size_t live = r->get_live_data_bytes(); + size_t garbage = r->garbage(); + if (r->is_young()) { _young_region_count++; - _young_bytes_to_evacuate += r->get_live_data_bytes(); + _young_bytes_to_evacuate += live; if (r->age() >= InitialTenuringThreshold) { - _young_bytes_to_promote += r->get_live_data_bytes(); + _young_bytes_to_promote += live; } } else if (r->is_old()) { _old_region_count++; - _old_bytes_to_evacuate += r->get_live_data_bytes(); - _old_garbage += r->garbage(); + _old_bytes_to_evacuate += live; + _old_garbage += garbage; } _region_count++; _has_old_regions |= r->is_old(); - _garbage += r->garbage(); + _garbage += garbage; _used += r->used(); - _live += r->get_live_data_bytes(); + _live += live; // Update the region status too. State transition would be checked internally. r->make_cset(); } @@ -184,8 +187,8 @@ void ShenandoahCollectionSet::print_on(outputStream* out) const { out->print_cr("Collection Set: Regions: " SIZE_FORMAT ", Garbage: " SIZE_FORMAT "%s, Live: " SIZE_FORMAT "%s, Used: " SIZE_FORMAT "%s", count(), byte_size_in_proper_unit(garbage()), proper_unit_for_byte_size(garbage()), - byte_size_in_proper_unit(live()), proper_unit_for_byte_size(live()), - byte_size_in_proper_unit(used()), proper_unit_for_byte_size(used())); + byte_size_in_proper_unit(live()), proper_unit_for_byte_size(live()), + byte_size_in_proper_unit(used()), proper_unit_for_byte_size(used())); debug_only(size_t regions = 0;) for (size_t index = 0; index < _heap->num_regions(); index ++) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp index 5b497ef50519e..112fff383ba25 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp @@ -58,11 +58,13 @@ class ShenandoahCollectionSet : public CHeapObj { size_t _young_region_count; size_t _old_region_count; - size_t _old_garbage; // How many bytes of old garbage are present in a mixed collection set? + // How many bytes of old garbage are present in a mixed collection set? + size_t _old_garbage; - bool* _preselected_regions; // Points to array identifying which tenure-age regions have been preselected - // for inclusion in collection set. This field is only valid during brief - // spans of time while collection set is being constructed. + // Points to array identifying which tenure-age regions have been preselected + // for inclusion in collection set. This field is only valid during brief + // spans of time while collection set is being constructed. + bool* _preselected_regions; shenandoah_padding(0); volatile size_t _current_index; @@ -100,17 +102,16 @@ class ShenandoahCollectionSet : public CHeapObj { // This represents total amount of work to be performed by evacuation, including evacuations to young, to old, // and promotions from young to old. This equals get_young_bytes_reserved_for_evacuation() plus // get_old_bytes_reserved_for_evacuation(). + // TODO: Seems unused. inline size_t get_bytes_reserved_for_evacuation(); // It is not known how many of these bytes will be promoted. inline size_t get_young_bytes_reserved_for_evacuation(); - inline size_t get_old_bytes_reserved_for_evacuation(); inline size_t get_young_bytes_to_be_promoted(); inline size_t get_old_region_count(); - inline size_t get_young_region_count(); inline size_t get_old_garbage(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index bfb0bbd6b384c..3a60cdd3798f3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -116,15 +116,14 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) { { ShenandoahBreakpointMarkScope breakpoint_mark_scope(cause); - // Reset task queue stats here, rather than in mark_concurrent_roots + // Reset task queue stats here, rather than in mark_concurrent_roots, // because remembered set scan will `push` oops into the queues and // resetting after this happens will lose those counts. TASKQUEUE_STATS_ONLY(_mark.task_queues()->reset_taskqueue_stats()); // Concurrent remembered set scanning entry_scan_remembered_set(); - // When RS scanning yields, we will need a check_cancellation_and_abort() - // degeneration point here. + // TODO: When RS scanning yields, we will need a check_cancellation_and_abort() degeneration point here. // Concurrent mark roots entry_mark_roots(); @@ -219,7 +218,6 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) { } if (heap->mode()->is_generational()) { - size_t old_available, young_available; { ShenandoahYoungGeneration* young_gen = heap->young_generation(); ShenandoahGeneration* old_gen = heap->old_generation(); @@ -235,9 +233,6 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) { // No need to old_gen->increase_used(). // That was done when plabs were allocated, accounting for both old evacs and promotions. - young_available = young_gen->adjusted_available(); - old_available = old_gen->adjusted_available(); - heap->set_alloc_supplement_reserve(0); heap->set_young_evac_reserve(0); heap->set_old_evac_reserve(0); @@ -622,12 +617,13 @@ void ShenandoahConcurrentGC::op_init_mark() { if (heap->mode()->is_generational()) { - if (_generation->is_young() || (_generation->is_global() && ShenandoahVerify)) { + if (_generation->is_young() || (_generation->is_global() && ShenandoahVerify)) { // The current implementation of swap_remembered_set() copies the write-card-table // to the read-card-table. The remembered sets are also swapped for GLOBAL collections // so that the verifier works with the correct copy of the card table when verifying. - ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_swap_rset); - _generation->swap_remembered_set(); + // TODO: This path should not really depend on ShenandoahVerify. + ShenandoahGCPhase phase(ShenandoahPhaseTimings::init_swap_rset); + _generation->swap_remembered_set(); } if (_generation->is_global()) { @@ -776,14 +772,15 @@ void ShenandoahConcurrentGC::op_final_mark() { if (heap->mode()->is_generational()) { // Calculate the temporary evacuation allowance supplement to young-gen memory capacity (for allocations // and young-gen evacuations). - size_t young_available = heap->young_generation()->adjust_available(heap->get_alloc_supplement_reserve()); + intptr_t adjustment = heap->get_alloc_supplement_reserve(); + size_t young_available = heap->young_generation()->adjust_available(adjustment); // old_available is memory that can hold promotions and evacuations. Subtract out the memory that is being // loaned for young-gen allocations or evacuations. - size_t old_available = heap->old_generation()->adjust_available(-heap->get_alloc_supplement_reserve()); + size_t old_available = heap->old_generation()->adjust_available(-adjustment); log_info(gc, ergo)("After generational memory budget adjustments, old available: " SIZE_FORMAT "%s, young_available: " SIZE_FORMAT "%s", - byte_size_in_proper_unit(old_available), proper_unit_for_byte_size(old_available), + byte_size_in_proper_unit(old_available), proper_unit_for_byte_size(old_available), byte_size_in_proper_unit(young_available), proper_unit_for_byte_size(young_available)); } @@ -1242,7 +1239,7 @@ void ShenandoahConcurrentGC::init_mark_event_message(char* buf, size_t len) cons void ShenandoahConcurrentGC::final_mark_event_message(char* buf, size_t len) const { ShenandoahHeap* const heap = ShenandoahHeap::heap(); assert(!heap->has_forwarded_objects() || heap->is_concurrent_old_mark_in_progress(), - "Should not have forwarded objects during final mark (unless old gen concurrent mark is running)"); + "Should not have forwarded objects during final mark, unless old gen concurrent mark is running"); if (heap->unload_classes()) { jio_snprintf(buf, len, "Pause Final Mark (%s) (unload classes)", _generation->name()); } else { @@ -1253,7 +1250,7 @@ void ShenandoahConcurrentGC::final_mark_event_message(char* buf, size_t len) con void ShenandoahConcurrentGC::conc_mark_event_message(char* buf, size_t len) const { ShenandoahHeap* const heap = ShenandoahHeap::heap(); assert(!heap->has_forwarded_objects() || heap->is_concurrent_old_mark_in_progress(), - "Should not have forwarded objects concurrent mark (unless old gen concurrent mark is running"); + "Should not have forwarded objects concurrent mark, unless old gen concurrent mark is running"); if (heap->unload_classes()) { jio_snprintf(buf, len, "Concurrent marking (%s) (unload classes)", _generation->name()); } else { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp index 2ce7185d3cf51..c7d64b70fef60 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp @@ -97,7 +97,6 @@ class ShenandoahConcurrentGC : public ShenandoahGC { void entry_class_unloading(); void entry_strong_roots(); void entry_cleanup_early(); - virtual void op_final_mark(); private: void entry_evacuate(); @@ -125,11 +124,13 @@ class ShenandoahConcurrentGC : public ShenandoahGC { void op_final_roots(); void op_cleanup_complete(); void op_global_coalesce_and_fill(); +protected: + virtual void op_final_mark(); +private: void start_mark(); - // Messages for GC trace events, they have to be immortal for - // passing around the logging/tracing systems + // Messages for GC trace events void init_mark_event_message(char* buf, size_t len) const; void final_mark_event_message(char* buf, size_t len) const; void conc_mark_event_message(char* buf, size_t len) const; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp index 11e6be2eb7b25..d14c6efe6dd59 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp @@ -118,14 +118,14 @@ class ShenandoahFinalMarkingTask : public WorkerTask { // First drain remaining SATB buffers. { ShenandoahObjToScanQueue* q = _cm->get_queue(worker_id); - ShenandoahObjToScanQueue* old = _cm->get_old_queue(worker_id); + ShenandoahObjToScanQueue* old_q = _cm->get_old_queue(worker_id); - ShenandoahSATBBufferClosure cl(q, old); + ShenandoahSATBBufferClosure cl(q, old_q); SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set(); while (satb_mq_set.apply_closure_to_completed_buffer(&cl)) {} assert(!heap->has_forwarded_objects(), "Not expected"); - ShenandoahMarkRefsClosure mark_cl(q, rp, old); + ShenandoahMarkRefsClosure mark_cl(q, rp, old_q); ShenandoahSATBAndRemarkThreadsClosure tc(satb_mq_set, ShenandoahIUBarrier ? &mark_cl : nullptr); Threads::possibly_parallel_threads_do(true /* is_par */, &tc); @@ -178,8 +178,9 @@ template void ShenandoahMarkConcurrentRootsTask::work(uint worker_id) { ShenandoahConcurrentWorkerSession worker_session(worker_id); ShenandoahObjToScanQueue* q = _queue_set->queue(worker_id); - ShenandoahObjToScanQueue* old = _old_queue_set == nullptr ? nullptr : _old_queue_set->queue(worker_id); - ShenandoahMarkRefsClosure cl(q, _rp, old); + ShenandoahObjToScanQueue* old_q = (_old_queue_set == nullptr) ? + nullptr : _old_queue_set->queue(worker_id); + ShenandoahMarkRefsClosure cl(q, _rp, old_q); _root_scanner.roots_do(&cl, worker_id); } @@ -192,19 +193,24 @@ void ShenandoahConcurrentMark::mark_concurrent_roots() { _generation->reserve_task_queues(workers->active_workers()); switch (_generation->type()) { case YOUNG: { - ShenandoahMarkConcurrentRootsTask task(task_queues(), old_task_queues(), rp, ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers()); + ShenandoahMarkConcurrentRootsTask task(task_queues(), old_task_queues(), rp, + ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers()); workers->run_task(&task); break; } case GLOBAL: { - assert(old_task_queues() == nullptr, "Global mark should not have old gen mark queues."); - ShenandoahMarkConcurrentRootsTask task(task_queues(), old_task_queues(), rp, ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers()); + assert(old_task_queues() == nullptr, "Global mark should not have old gen mark queues"); + ShenandoahMarkConcurrentRootsTask task(task_queues(), nullptr, rp, + ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers()); workers->run_task(&task); break; } + case OLD: { + // We use a YOUNG generation cycle to bootstrap concurrent old marking. + ShouldNotReachHere(); + break; + } default: - // Intentionally haven't added OLD here. We use a YOUNG generation - // cycle to bootstrap concurrent old marking. ShouldNotReachHere(); } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp index 6f9b402cdd144..625ccc9271e39 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp @@ -30,7 +30,7 @@ template class ShenandoahConcurrentMarkingTask; -template +template class ShenandoahFinalMarkingTask; class ShenandoahGeneration; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index 704b7060e5ca8..b10e4b98058cd 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -142,16 +142,17 @@ void ShenandoahControlThread::run_service() { _degen_point = ShenandoahGC::_degenerated_outside_cycle; if (degen_point == ShenandoahGC::_degenerated_outside_cycle) { - _degen_generation = heap->mode()->is_generational() ? heap->young_generation() : heap->global_generation(); + _degen_generation = heap->mode()->is_generational() ? + heap->young_generation() : heap->global_generation(); } else { - assert(_degen_generation != nullptr, "Need to know which generation to resume."); + assert(_degen_generation != nullptr, "Need to know which generation to resume"); } ShenandoahHeuristics* heuristics = _degen_generation->heuristics(); generation = _degen_generation->type(); bool old_gen_evacuation_failed = heap->clear_old_evacuation_failure(); - // Do not bother with degenerated cycle if old generation evacuation failed. + // Do not bother with degenerated cycle if old generation evacuation failed if (ShenandoahDegeneratedGC && heuristics->should_degenerate_cycle() && !old_gen_evacuation_failed) { heuristics->record_allocation_failure_gc(); policy->record_alloc_failure_to_degenerated(degen_point); @@ -203,7 +204,7 @@ void ShenandoahControlThread::run_service() { // If a request to start an old cycle arrived while an old cycle was running, but _before_ // it chose any regions for evacuation we don't want to start a new old cycle. Rather, we want // the heuristic to run a young collection so that we can evacuate some old regions. - assert(!heap->is_concurrent_old_mark_in_progress(), "Should not be running mixed collections and concurrent marking."); + assert(!heap->is_concurrent_old_mark_in_progress(), "Should not be running mixed collections and concurrent marking"); generation = YOUNG; } else { generation = _requested_generation; @@ -213,7 +214,7 @@ void ShenandoahControlThread::run_service() { cause = GCCause::_shenandoah_concurrent_gc; set_gc_mode(default_mode); - // Don't start a new old marking if there is one already in progress. + // Don't start a new old marking if there is one already in progress if (generation == OLD && heap->is_concurrent_old_mark_in_progress()) { set_gc_mode(servicing_old); } @@ -232,9 +233,9 @@ void ShenandoahControlThread::run_service() { } else if (heap->is_concurrent_old_mark_in_progress() || heap->is_prepare_for_old_mark_in_progress()) { // Nobody asked us to do anything, but we have an old-generation mark or old-generation preparation for // mixed evacuation in progress, so resume working on that. - log_info(gc)("Resume old gc: marking=%s, preparing=%s", - BOOL_TO_STR(heap->is_concurrent_old_mark_in_progress()), - BOOL_TO_STR(heap->is_prepare_for_old_mark_in_progress())); + log_info(gc)("Resume old GC: marking is%s in progress, preparing is%s in progress", + heap->is_concurrent_old_mark_in_progress() ? "" : " NOT", + heap->is_prepare_for_old_mark_in_progress() ? "" : " NOT"); cause = GCCause::_shenandoah_concurrent_gc; generation = OLD; @@ -271,47 +272,45 @@ void ShenandoahControlThread::run_service() { // In case this is a degenerated cycle, remember whether original cycle was aging. bool was_aging_cycle = heap->is_aging_cycle(); heap->set_aging_cycle(false); - { - switch (_mode) { - case concurrent_normal: { - // At this point: - // if (generation == YOUNG), this is a normal YOUNG cycle - // if (generation == OLD), this is a bootstrap OLD cycle - // if (generation == GLOBAL), this is a GLOBAL cycle triggered by System.gc() - // In all three cases, we want to age old objects if this is an aging cycle - if (age_period-- == 0) { - heap->set_aging_cycle(true); - age_period = ShenandoahAgingCyclePeriod - 1; - } - service_concurrent_normal_cycle(heap, generation, cause); - break; - } - case stw_degenerated: { - heap->set_aging_cycle(was_aging_cycle); - if (!service_stw_degenerated_cycle(cause, degen_point)) { - // The degenerated GC was upgraded to a Full GC - generation = GLOBAL; - } - break; - } - case stw_full: { - if (age_period-- == 0) { - heap->set_aging_cycle(true); - age_period = ShenandoahAgingCyclePeriod - 1; - } - service_stw_full_cycle(cause); - break; + + switch (_mode) { + case concurrent_normal: { + // At this point: + // if (generation == YOUNG), this is a normal YOUNG cycle + // if (generation == OLD), this is a bootstrap OLD cycle + // if (generation == GLOBAL), this is a GLOBAL cycle triggered by System.gc() + // In all three cases, we want to age old objects if this is an aging cycle + if (age_period-- == 0) { + heap->set_aging_cycle(true); + age_period = ShenandoahAgingCyclePeriod - 1; } - case servicing_old: { - assert(generation == OLD, "Expected old generation here"); - GCIdMark gc_id_mark; - service_concurrent_old_cycle(heap, cause); - break; + service_concurrent_normal_cycle(heap, generation, cause); + break; + } + case stw_degenerated: { + heap->set_aging_cycle(was_aging_cycle); + if (!service_stw_degenerated_cycle(cause, degen_point)) { + // The degenerated GC was upgraded to a Full GC + generation = GLOBAL; } - default: { - ShouldNotReachHere(); + break; + } + case stw_full: { + if (age_period-- == 0) { + heap->set_aging_cycle(true); + age_period = ShenandoahAgingCyclePeriod - 1; } + service_stw_full_cycle(cause); + break; + } + case servicing_old: { + assert(generation == OLD, "Expected old generation here"); + GCIdMark gc_id_mark; + service_concurrent_old_cycle(heap, cause); + break; } + default: + ShouldNotReachHere(); } // If this was the requested GC cycle, notify waiters about it @@ -405,7 +404,6 @@ void ShenandoahControlThread::run_service() { } void ShenandoahControlThread::process_phase_timings(const ShenandoahHeap* heap) { - // Commit worker statistics to cycle data heap->phase_timings()->flush_par_workers_to_cycle(); if (ShenandoahPacing) { @@ -422,7 +420,7 @@ void ShenandoahControlThread::process_phase_timings(const ShenandoahHeap* heap) LogStream ls(lt); heap->phase_timings()->print_cycle_on(&ls); ShenandoahEvacuationTracker::print_evacuations_on(&ls, &evac_stats.workers, - &evac_stats.mutators); + &evac_stats.mutators); if (ShenandoahPacing) { heap->pacer()->print_cycle_on(&ls); } @@ -431,7 +429,6 @@ void ShenandoahControlThread::process_phase_timings(const ShenandoahHeap* heap) // Commit statistics to globals heap->phase_timings()->flush_cycle_to_global(); - } // Young and old concurrent cycles are initiated by the regulator. Implicit @@ -459,8 +456,9 @@ void ShenandoahControlThread::process_phase_timings(const ShenandoahHeap* heap) // | v v | // +---> Global Degen +--------------------> Full <----+ // -void ShenandoahControlThread::service_concurrent_normal_cycle( - const ShenandoahHeap* heap, const ShenandoahGenerationType generation, GCCause::Cause cause) { +void ShenandoahControlThread::service_concurrent_normal_cycle(const ShenandoahHeap* heap, + const ShenandoahGenerationType generation, + GCCause::Cause cause) { GCIdMark gc_id_mark; switch (generation) { case YOUNG: { @@ -489,18 +487,20 @@ void ShenandoahControlThread::service_concurrent_normal_cycle( const char* msg; if (heap->mode()->is_generational()) { if (heap->cancelled_gc()) { - msg = (generation == YOUNG)? "At end of Interrupted Concurrent Young GC": "At end of Interrupted Concurrent Bootstrap GC"; + msg = (generation == YOUNG) ? "At end of Interrupted Concurrent Young GC" : + "At end of Interrupted Concurrent Bootstrap GC"; } else { - msg = (generation == YOUNG)? "At end of Concurrent Young GC": "At end of Concurrent Bootstrap GC"; + msg = (generation == YOUNG) ? "At end of Concurrent Young GC" : + "At end of Concurrent Bootstrap GC"; } } else { - msg = heap->cancelled_gc() ? "At end of cancelled GC" : "At end of GC"; + msg = heap->cancelled_gc() ? "At end of cancelled GC" : + "At end of GC"; } heap->log_heap_status(msg); } void ShenandoahControlThread::service_concurrent_old_cycle(const ShenandoahHeap* heap, GCCause::Cause &cause) { - ShenandoahOldGeneration* old_generation = heap->old_generation(); ShenandoahYoungGeneration* young_generation = heap->young_generation(); ShenandoahOldGeneration::State original_state = old_generation->state(); @@ -510,8 +510,8 @@ void ShenandoahControlThread::service_concurrent_old_cycle(const ShenandoahHeap* switch (original_state) { case ShenandoahOldGeneration::WAITING_FOR_FILL: case ShenandoahOldGeneration::IDLE: { - assert(!heap->is_concurrent_old_mark_in_progress(), "Old already in progress."); - assert(old_generation->task_queues()->is_empty(), "Old mark queues should be empty."); + assert(!heap->is_concurrent_old_mark_in_progress(), "Old already in progress"); + assert(old_generation->task_queues()->is_empty(), "Old mark queues should be empty"); } case ShenandoahOldGeneration::FILLING: { _allow_old_preemption.set(); @@ -520,11 +520,11 @@ void ShenandoahControlThread::service_concurrent_old_cycle(const ShenandoahHeap* _allow_old_preemption.unset(); if (heap->is_prepare_for_old_mark_in_progress()) { - assert(old_generation->state() == ShenandoahOldGeneration::FILLING, "Prepare for mark should be in progress."); + assert(old_generation->state() == ShenandoahOldGeneration::FILLING, "Prepare for mark should be in progress"); return; } - assert(old_generation->state() == ShenandoahOldGeneration::BOOTSTRAPPING, "Finished with filling, should be bootstrapping."); + assert(old_generation->state() == ShenandoahOldGeneration::BOOTSTRAPPING, "Finished with filling, should be bootstrapping"); } case ShenandoahOldGeneration::BOOTSTRAPPING: { // Configure the young generation's concurrent mark to put objects in @@ -539,7 +539,7 @@ void ShenandoahControlThread::service_concurrent_old_cycle(const ShenandoahHeap* if (heap->cancelled_gc()) { // Young generation bootstrap cycle has failed. Concurrent mark for old generation // is going to resume after degenerated bootstrap cycle completes. - log_info(gc)("Bootstrap cycle for old generation was cancelled."); + log_info(gc)("Bootstrap cycle for old generation was cancelled"); return; } @@ -560,7 +560,7 @@ void ShenandoahControlThread::service_concurrent_old_cycle(const ShenandoahHeap* ShenandoahGCSession session(cause, old_generation); bool marking_complete = resume_concurrent_old_cycle(old_generation, cause); if (marking_complete) { - assert(old_generation->state() != ShenandoahOldGeneration::MARKING, "Should not still be marking."); + assert(old_generation->state() != ShenandoahOldGeneration::MARKING, "Should not still be marking"); if (original_state == ShenandoahOldGeneration::MARKING) { heap->log_heap_status("At end of Concurrent Old Marking finishing increment"); } @@ -570,15 +570,13 @@ void ShenandoahControlThread::service_concurrent_old_cycle(const ShenandoahHeap* break; } default: - log_error(gc)("Unexpected state for old GC: %d", old_generation->state()); - ShouldNotReachHere(); + fatal("Unexpected state for old GC: %s", ShenandoahOldGeneration::state_name(old_generation->state())); } } bool ShenandoahControlThread::resume_concurrent_old_cycle(ShenandoahGeneration* generation, GCCause::Cause cause) { - assert(ShenandoahHeap::heap()->is_concurrent_old_mark_in_progress(), "Old mark should be in progress"); - log_debug(gc)("Resuming old generation with " UINT32_FORMAT " marking tasks queued.", generation->task_queues()->tasks()); + log_debug(gc)("Resuming old generation with " UINT32_FORMAT " marking tasks queued", generation->task_queues()->tasks()); ShenandoahHeap* heap = ShenandoahHeap::heap(); @@ -673,8 +671,10 @@ void ShenandoahControlThread::service_concurrent_cycle(ShenandoahGeneration* gen service_concurrent_cycle(heap, generation, cause, do_old_gc_bootstrap); } -void ShenandoahControlThread::service_concurrent_cycle(const ShenandoahHeap* heap, ShenandoahGeneration* generation, - GCCause::Cause &cause, bool do_old_gc_bootstrap) { +void ShenandoahControlThread::service_concurrent_cycle(const ShenandoahHeap* heap, + ShenandoahGeneration* generation, + GCCause::Cause& cause, + bool do_old_gc_bootstrap) { ShenandoahConcurrentGC gc(generation, do_old_gc_bootstrap); if (gc.collect(cause)) { // Cycle is complete @@ -721,7 +721,7 @@ bool ShenandoahControlThread::check_cancellation_or_degen(ShenandoahGC::Shenando return true; } - fatal("Cancel GC either for alloc failure GC, or gracefully exiting, or to pause old generation marking."); + fatal("Cancel GC either for alloc failure GC, or gracefully exiting, or to pause old generation marking"); return false; } @@ -742,8 +742,9 @@ void ShenandoahControlThread::service_stw_full_cycle(GCCause::Cause cause) { heap->shenandoah_policy()->record_success_full(); } -bool ShenandoahControlThread::service_stw_degenerated_cycle(GCCause::Cause cause, ShenandoahGC::ShenandoahDegenPoint point) { - assert (point != ShenandoahGC::_degenerated_unset, "Degenerated point should be set"); +bool ShenandoahControlThread::service_stw_degenerated_cycle(GCCause::Cause cause, + ShenandoahGC::ShenandoahDegenPoint point) { + assert(point != ShenandoahGC::_degenerated_unset, "Degenerated point should be set"); ShenandoahHeap* const heap = ShenandoahHeap::heap(); GCIdMark gc_id_mark; @@ -758,9 +759,9 @@ bool ShenandoahControlThread::service_stw_degenerated_cycle(GCCause::Cause cause assert(heap->global_generation()->task_queues()->is_empty(), "Unexpected global generation marking tasks"); } else { assert(_degen_generation->is_young(), "Expected degenerated young cycle, if not global."); - ShenandoahOldGeneration* old_generation = (ShenandoahOldGeneration*) heap->old_generation(); - if (old_generation->state() == ShenandoahOldGeneration::BOOTSTRAPPING && !gc.upgraded_to_full()) { - old_generation->transition_to(ShenandoahOldGeneration::MARKING); + ShenandoahOldGeneration* old = heap->old_generation(); + if (old->state() == ShenandoahOldGeneration::BOOTSTRAPPING && !gc.upgraded_to_full()) { + old->transition_to(ShenandoahOldGeneration::MARKING); } } @@ -798,7 +799,8 @@ bool ShenandoahControlThread::is_explicit_gc(GCCause::Cause cause) const { } bool ShenandoahControlThread::is_implicit_gc(GCCause::Cause cause) const { - return !is_explicit_gc(cause) && cause != GCCause::_shenandoah_concurrent_gc; + return !is_explicit_gc(cause) && + (cause != GCCause::_shenandoah_concurrent_gc); } void ShenandoahControlThread::request_gc(GCCause::Cause cause) { @@ -825,7 +827,7 @@ void ShenandoahControlThread::request_gc(GCCause::Cause cause) { bool ShenandoahControlThread::request_concurrent_gc(ShenandoahGenerationType generation) { if (_preemption_requested.is_set() || _gc_requested.is_set() || ShenandoahHeap::heap()->cancelled_gc()) { - // ignore subsequent requests from the heuristics + // Ignore subsequent requests from the heuristics return false; } @@ -839,7 +841,7 @@ bool ShenandoahControlThread::request_concurrent_gc(ShenandoahGenerationType gen } if (preempt_old_marking(generation)) { - log_info(gc)("Preempting old generation mark to allow %s GC.", shenandoah_generation_name(generation)); + log_info(gc)("Preempting old generation mark to allow %s GC", shenandoah_generation_name(generation)); _requested_gc_cause = GCCause::_shenandoah_concurrent_gc; _requested_generation = generation; _preemption_requested.set(); @@ -860,7 +862,7 @@ void ShenandoahControlThread::notify_control_thread() { } bool ShenandoahControlThread::preempt_old_marking(ShenandoahGenerationType generation) { - return generation == YOUNG && _allow_old_preemption.try_unset(); + return (generation == YOUNG) && _allow_old_preemption.try_unset(); } void ShenandoahControlThread::handle_requested_gc(GCCause::Cause cause) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp index e8c500b7dcbf0..7c937286d53ac 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp @@ -188,7 +188,9 @@ class ShenandoahControlThread: public ConcurrentGCThread { static const char* gc_mode_name(GCMode mode); void notify_control_thread(); - void service_concurrent_cycle(const ShenandoahHeap* heap, ShenandoahGeneration* generation, GCCause::Cause &cause, + void service_concurrent_cycle(const ShenandoahHeap* heap, + ShenandoahGeneration* generation, + GCCause::Cause &cause, bool do_old_gc_bootstrap); }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp index 1c3732b9cdc2a..ef288048f4ee4 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp @@ -97,12 +97,12 @@ void ShenandoahDegenGC::op_degenerated() { // We can only get to a degenerated global cycle _after_ a concurrent global cycle // has been cancelled. In which case, we expect the concurrent global cycle to have // cancelled the old gc already. - assert(!heap->is_old_gc_active(), "Old GC should not be active during global cycle."); + assert(!heap->is_old_gc_active(), "Old GC should not be active during global cycle"); } if (!heap->is_concurrent_old_mark_in_progress()) { // If we are not marking the old generation, there should be nothing in the old mark queues - assert(heap->old_generation()->task_queues()->is_empty(), "Old gen task queues should be empty."); + assert(heap->old_generation()->task_queues()->is_empty(), "Old gen task queues should be empty"); } } #endif @@ -133,10 +133,13 @@ void ShenandoahDegenGC::op_degenerated() { // Note that we can only do this for "outside-cycle" degens, otherwise we would risk // changing the cycle parameters mid-cycle during concurrent -> degenerated handover. - heap->set_unload_classes((!heap->mode()->is_generational() || _generation->is_global()) && _generation->heuristics()->can_unload_classes()); + heap->set_unload_classes(_generation->heuristics()->can_unload_classes() && + (!heap->mode()->is_generational() || _generation->is_global())); - if (heap->mode()->is_generational() && (_generation->is_young() || (_generation->is_global() && ShenandoahVerify))) { + if (heap->mode()->is_generational() && + (_generation->is_young() || (_generation->is_global() && ShenandoahVerify))) { // Swap remembered sets for young, or if the verifier will run during a global collect + // TODO: This path should not depend on ShenandoahVerify _generation->swap_remembered_set(); } @@ -276,12 +279,13 @@ void ShenandoahDegenGC::op_degenerated() { } if (heap->mode()->is_generational()) { - // In case degeneration interrupted concurrent evacuation or update references, we need to clean up transient state. - // Otherwise, these actions have no effect. + // In case degeneration interrupted concurrent evacuation or update references, + // we need to clean up transient state. Otherwise, these actions have no effect. heap->young_generation()->unadjust_available(); heap->old_generation()->unadjust_available(); - // No need to old_gen->increase_used(). That was done when plabs were allocated, accounting for both old evacs and promotions. + // No need to old_gen->increase_used(). That was done when plabs were allocated, + // accounting for both old evacs and promotions. heap->set_alloc_supplement_reserve(0); heap->set_young_evac_reserve(0); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.cpp b/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.cpp index a49d8a13a8b7e..8eb3ba9497154 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.cpp @@ -44,7 +44,7 @@ void ShenandoahEvacuationStats::end_evacuation(size_t bytes, uint age) { ++_evacuations_completed; _bytes_completed += bytes; if (age > 0) { - _age_table.add(age, bytes / oopSize); + _age_table.add(age, bytes >> LogBytesPerWord); } } @@ -68,10 +68,10 @@ void ShenandoahEvacuationStats::print_on(outputStream* st) { size_t abandoned_count = _evacuations_attempted - _evacuations_completed; st->print_cr("Evacuated " SIZE_FORMAT "%s across " SIZE_FORMAT " objects, " "abandoned " SIZE_FORMAT "%s across " SIZE_FORMAT " objects.", - byte_size_in_proper_unit(_bytes_completed), - proper_unit_for_byte_size(_bytes_completed), _evacuations_completed, - byte_size_in_proper_unit(abandoned_size), - proper_unit_for_byte_size(abandoned_size), abandoned_count); + byte_size_in_proper_unit(_bytes_completed), proper_unit_for_byte_size(_bytes_completed), + _evacuations_completed, + byte_size_in_proper_unit(abandoned_size), proper_unit_for_byte_size(abandoned_size), + abandoned_count); _age_table.print_on(st, InitialTenuringThreshold); } @@ -97,6 +97,7 @@ void ShenandoahEvacuationTracker::print_evacuations_on(outputStream* st, if (r->is_young()) { young_region_ages.add(r->age(), r->get_live_data_words()); } else { + assert(r->is_old(), "Sanity"); old_region_ages.add(r->age(), r->get_live_data_words()); } } @@ -109,7 +110,7 @@ void ShenandoahEvacuationTracker::print_evacuations_on(outputStream* st, } class ShenandoahStatAggregator : public ThreadClosure { - public: +public: ShenandoahEvacuationStats* _target; explicit ShenandoahStatAggregator(ShenandoahEvacuationStats* target) : _target(target) {} virtual void do_thread(Thread* thread) override { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.hpp b/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.hpp index 9a864cf3a0581..72729cd9e1876 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahEvacTracker.hpp @@ -29,7 +29,7 @@ #include "utilities/ostream.hpp" class ShenandoahEvacuationStats : public CHeapObj { - private: +private: size_t _evacuations_completed; size_t _bytes_completed; size_t _evacuations_attempted; @@ -37,7 +37,7 @@ class ShenandoahEvacuationStats : public CHeapObj { AgeTable _age_table; - public: +public: ShenandoahEvacuationStats(); void begin_evacuation(size_t bytes); void end_evacuation(size_t bytes, uint age); @@ -53,19 +53,20 @@ struct ShenandoahCycleStats { }; class ShenandoahEvacuationTracker : public CHeapObj { - private: - ShenandoahEvacuationStats _workers_global; - ShenandoahEvacuationStats _mutators_global; +private: + ShenandoahEvacuationStats _workers_global; + ShenandoahEvacuationStats _mutators_global; - public: +public: void begin_evacuation(Thread* thread, size_t bytes); void end_evacuation(Thread* thread, size_t bytes, uint age); void print_global_on(outputStream* st); - static void print_evacuations_on(outputStream* st, ShenandoahEvacuationStats* workers, ShenandoahEvacuationStats* mutators); + static void print_evacuations_on(outputStream* st, + ShenandoahEvacuationStats* workers, + ShenandoahEvacuationStats* mutators); ShenandoahCycleStats flush_cycle_to_global(); - private: }; #endif //SHARE_GC_SHENANDOAH_SHENANDOAHEVACTRACKER_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index e0126ba43f9c8..040527033cecd 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -145,6 +145,8 @@ HeapWord* ShenandoahFreeSet::allocate_single(ShenandoahAllocRequest& req, bool& break; case ShenandoahAffiliation::FREE: + fatal("Should request affiliation"); + default: ShouldNotReachHere(); break; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index ec91e2d6a8fda..d2aa7a1fcaede 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -91,10 +91,9 @@ class ShenandoahReconstructRememberedSetTask : public WorkerTask { // First, clear the remembered set oop obj = cast_to_oop(obj_addr); size_t size = obj->size(); - HeapWord* end_object = r->bottom() + size; // First, clear the remembered set for all spanned humongous regions - size_t num_regions = (size + ShenandoahHeapRegion::region_size_words() - 1) / ShenandoahHeapRegion::region_size_words(); + size_t num_regions = ShenandoahHeapRegion::required_regions(size * HeapWordSize); size_t region_span = num_regions * ShenandoahHeapRegion::region_size_words(); scanner->reset_remset(r->bottom(), region_span); size_t region_index = r->index(); @@ -106,7 +105,7 @@ class ShenandoahReconstructRememberedSetTask : public WorkerTask { } // Then register the humongous object and DIRTY relevant remembered set cards - scanner->register_object_wo_lock(obj_addr); + scanner->register_object_without_lock(obj_addr); obj->oop_iterate(&dirty_cards_for_interesting_pointers); } else if (!r->is_humongous()) { // First, clear the remembered set @@ -118,12 +117,13 @@ class ShenandoahReconstructRememberedSetTask : public WorkerTask { while (obj_addr < t) { oop obj = cast_to_oop(obj_addr); size_t size = obj->size(); - scanner->register_object_wo_lock(obj_addr); + scanner->register_object_without_lock(obj_addr); obj_addr += obj->oop_iterate_size(&dirty_cards_for_interesting_pointers); } } // else, ignore humongous continuation region } // else, this region is FREE or YOUNG or inactive and we can ignore it. + // TODO: Assert this. r = _regions.next(); } } @@ -329,6 +329,7 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) { { // Epilogue + // TODO: Merge with phase5_epilog? _preserved_marks->restore(heap->workers()); _preserved_marks->reclaim(); @@ -420,7 +421,8 @@ class ShenandoahPrepareForCompactionTask : public WorkerTask { size_t const _num_workers; public: - ShenandoahPrepareForCompactionTask(PreservedMarksSet *preserved_marks, ShenandoahHeapRegionSet **worker_slices, + ShenandoahPrepareForCompactionTask(PreservedMarksSet *preserved_marks, + ShenandoahHeapRegionSet **worker_slices, size_t num_workers); static bool is_candidate_region(ShenandoahHeapRegion* r) { @@ -440,7 +442,6 @@ class ShenandoahPrepareForCompactionTask : public WorkerTask { class ShenandoahPrepareForGenerationalCompactionObjectClosure : public ObjectClosure { private: - ShenandoahPrepareForCompactionTask* _compactor; PreservedMarks* const _preserved_marks; ShenandoahHeap* const _heap; @@ -458,12 +459,10 @@ class ShenandoahPrepareForGenerationalCompactionObjectClosure : public ObjectClo uint _worker_id; public: - ShenandoahPrepareForGenerationalCompactionObjectClosure(ShenandoahPrepareForCompactionTask* compactor, - PreservedMarks* preserved_marks, + ShenandoahPrepareForGenerationalCompactionObjectClosure(PreservedMarks* preserved_marks, GrowableArray& empty_regions, ShenandoahHeapRegion* old_to_region, ShenandoahHeapRegion* young_to_region, uint worker_id) : - _compactor(compactor), _preserved_marks(preserved_marks), _heap(ShenandoahHeap::heap()), _empty_regions(empty_regions), @@ -745,8 +744,10 @@ void ShenandoahPrepareForCompactionTask::work(uint worker_id) { if (_heap->mode()->is_generational()) { ShenandoahHeapRegion* old_to_region = (from_region->is_old())? from_region: nullptr; ShenandoahHeapRegion* young_to_region = (from_region->is_young())? from_region: nullptr; - ShenandoahPrepareForGenerationalCompactionObjectClosure cl(this, _preserved_marks->get(worker_id), empty_regions, - old_to_region, young_to_region, worker_id); + ShenandoahPrepareForGenerationalCompactionObjectClosure cl(_preserved_marks->get(worker_id), + empty_regions, + old_to_region, young_to_region, + worker_id); while (from_region != nullptr) { assert(is_candidate_region(from_region), "Sanity"); log_debug(gc)("Worker %u compacting %s Region " SIZE_FORMAT " which had used " SIZE_FORMAT " and %s live", @@ -854,7 +855,7 @@ class ShenandoahEnsureHeapActiveClosure: public ShenandoahHeapRegionClosure { r->recycle(); } if (r->is_cset()) { - // Leave afffiliation unchanged. + // Leave affiliation unchanged r->make_regular_bypass(); } if (r->is_empty_uncommitted()) { @@ -879,32 +880,34 @@ class ShenandoahTrashImmediateGarbageClosure: public ShenandoahHeapRegionClosure _ctx(ShenandoahHeap::heap()->complete_marking_context()) {} void heap_region_do(ShenandoahHeapRegion* r) { - if (r->affiliation() != FREE) { - if (r->is_humongous_start()) { - oop humongous_obj = cast_to_oop(r->bottom()); - if (!_ctx->is_marked(humongous_obj)) { - assert(!r->has_live(), - "Humongous Start %s Region " SIZE_FORMAT " is not marked, should not have live", - r->affiliation_name(), r->index()); - log_debug(gc)("Trashing immediate humongous region " SIZE_FORMAT " because not marked", r->index()); - _heap->trash_humongous_region_at(r); - } else { - assert(r->has_live(), - "Humongous Start %s Region " SIZE_FORMAT " should have live", r->affiliation_name(), r->index()); - } - } else if (r->is_humongous_continuation()) { - // If we hit continuation, the non-live humongous starts should have been trashed already - assert(r->humongous_start_region()->has_live(), - "Humongous Continuation %s Region " SIZE_FORMAT " should have live", r->affiliation_name(), r->index()); - } else if (r->is_regular()) { - if (!r->has_live()) { - log_debug(gc)("Trashing immediate regular region " SIZE_FORMAT " because has no live", r->index()); - r->make_trash_immediate(); - } + if (!r->is_affiliated()) { + // Ignore free regions + // TODO: change iterators so they do not process FREE regions. + return; + } + + if (r->is_humongous_start()) { + oop humongous_obj = cast_to_oop(r->bottom()); + if (!_ctx->is_marked(humongous_obj)) { + assert(!r->has_live(), + "Humongous Start %s Region " SIZE_FORMAT " is not marked, should not have live", + r->affiliation_name(), r->index()); + log_debug(gc)("Trashing immediate humongous region " SIZE_FORMAT " because not marked", r->index()); + _heap->trash_humongous_region_at(r); + } else { + assert(r->has_live(), + "Humongous Start %s Region " SIZE_FORMAT " should have live", r->affiliation_name(), r->index()); + } + } else if (r->is_humongous_continuation()) { + // If we hit continuation, the non-live humongous starts should have been trashed already + assert(r->humongous_start_region()->has_live(), + "Humongous Continuation %s Region " SIZE_FORMAT " should have live", r->affiliation_name(), r->index()); + } else if (r->is_regular()) { + if (!r->has_live()) { + log_debug(gc)("Trashing immediate regular region " SIZE_FORMAT " because has no live", r->index()); + r->make_trash_immediate(); } } - // else, ignore this FREE region. - // TODO: change iterators so they do not process FREE regions. } }; @@ -1159,7 +1162,7 @@ class ShenandoahAdjustPointersTask : public WorkerTask { // reference to reclaimed memory. Remembered set scanning will crash if it attempts // to iterate the oops in these objects. r->begin_preemptible_coalesce_and_fill(); - r->oop_fill_and_coalesce_wo_cancel(); + r->oop_fill_and_coalesce_without_cancel(); } r = _regions.next(); } @@ -1287,9 +1290,15 @@ class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure { size_t _old_regions, _old_usage, _old_humongous_waste; public: - ShenandoahPostCompactClosure() : _heap(ShenandoahHeap::heap()), _live(0), _is_generational(_heap->mode()->is_generational()), - _young_regions(0), _young_usage(0), _young_humongous_waste(0), - _old_regions(0), _old_usage(0), _old_humongous_waste(0) + ShenandoahPostCompactClosure() : _heap(ShenandoahHeap::heap()), + _live(0), + _is_generational(_heap->mode()->is_generational()), + _young_regions(0), + _young_usage(0), + _young_humongous_waste(0), + _old_regions(0), + _old_usage(0), + _old_humongous_waste(0) { _heap->free_set()->clear(); } @@ -1308,7 +1317,6 @@ class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure { size_t live = r->used(); - // Make empty regions that have been allocated into regular if (r->is_empty() && live > 0) { if (!_is_generational) { @@ -1332,6 +1340,8 @@ class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure { account_for_region(r, _old_regions, _old_usage, _old_humongous_waste); } else if (r->is_young()) { account_for_region(r, _young_regions, _young_usage, _young_humongous_waste); + } else { + // TODO: Assert here? } } @@ -1491,8 +1501,10 @@ void ShenandoahFullGC::phase5_epilog() { heap->set_used(post_compact.get_live()); if (heap->mode()->is_generational()) { post_compact.update_generation_usage(); - log_info(gc)("FullGC done: GLOBAL usage: " SIZE_FORMAT ", young usage: " SIZE_FORMAT ", old usage: " SIZE_FORMAT, - post_compact.get_live(), heap->young_generation()->used(), heap->old_generation()->used()); + log_info(gc)("FullGC done: GLOBAL usage: " SIZE_FORMAT "%s, young usage: " SIZE_FORMAT "%s, old usage: " SIZE_FORMAT "%s", + byte_size_in_proper_unit(post_compact.get_live()), proper_unit_for_byte_size(post_compact.get_live()), + byte_size_in_proper_unit(heap->young_generation()->used()), proper_unit_for_byte_size(heap->young_generation()->used()), + byte_size_in_proper_unit(heap->old_generation()->used()), proper_unit_for_byte_size(heap->old_generation()->used())); } heap->collection_set()->clear(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index d846321a34409..427dc41c969dd 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -167,12 +167,12 @@ void ShenandoahGeneration::log_status(const char *msg) const { "soft capacity: " SIZE_FORMAT "%s, max capacity: " SIZE_FORMAT "%s, available: " SIZE_FORMAT "%s, " "adjusted available: " SIZE_FORMAT "%s", msg, name(), - byte_size_in_proper_unit(v_used), proper_unit_for_byte_size(v_used), - byte_size_in_proper_unit(v_used_regions), proper_unit_for_byte_size(v_used_regions), + byte_size_in_proper_unit(v_used), proper_unit_for_byte_size(v_used), + byte_size_in_proper_unit(v_used_regions), proper_unit_for_byte_size(v_used_regions), byte_size_in_proper_unit(v_soft_max_capacity), proper_unit_for_byte_size(v_soft_max_capacity), - byte_size_in_proper_unit(v_max_capacity), proper_unit_for_byte_size(v_max_capacity), - byte_size_in_proper_unit(v_available), proper_unit_for_byte_size(v_available), - byte_size_in_proper_unit(v_adjusted_avail), proper_unit_for_byte_size(v_adjusted_avail)); + byte_size_in_proper_unit(v_max_capacity), proper_unit_for_byte_size(v_max_capacity), + byte_size_in_proper_unit(v_available), proper_unit_for_byte_size(v_available), + byte_size_in_proper_unit(v_adjusted_avail), proper_unit_for_byte_size(v_adjusted_avail)); } void ShenandoahGeneration::reset_mark_bitmap() { @@ -490,7 +490,7 @@ void ShenandoahGeneration::adjust_evacuation_budgets(ShenandoahHeap* heap, Shena old_regions_loaned_for_young_evac = revised_loan_for_young_evacuation; loaned_regions = old_regions_loaned_for_young_evac; } else { - // Undo the prevous loan, if any. + // Undo the previous loan, if any. old_regions_loaned_for_young_evac = 0; loaned_regions = 0; } @@ -518,8 +518,6 @@ void ShenandoahGeneration::adjust_evacuation_budgets(ShenandoahHeap* heap, Shena // to be an integral number of regions. size_t fragmented_old_usage = old_evacuated_committed + consumed_by_advance_promotion; - - if (fragmented_old_total >= fragmented_old_usage) { // Seems this will be rare. In this case, all of the memory required for old-gen evacuations and promotions can be // taken from the existing fragments within old-gen. Reduce this fragmented total by this amount. @@ -761,15 +759,20 @@ void ShenandoahGeneration::adjust_evacuation_budgets(ShenandoahHeap* heap, Shena "%s, loaned for young evacuation: " SIZE_FORMAT "%s, loaned for young allocations: " SIZE_FORMAT "%s, excess: " SIZE_FORMAT "%s", - byte_size_in_proper_unit(old_available), proper_unit_for_byte_size(old_available), - byte_size_in_proper_unit(old_evacuation_reserve), proper_unit_for_byte_size(old_evacuation_reserve), + byte_size_in_proper_unit(old_available), + proper_unit_for_byte_size(old_available), + byte_size_in_proper_unit(old_evacuation_reserve), + proper_unit_for_byte_size(old_evacuation_reserve), byte_size_in_proper_unit(consumed_by_advance_promotion), proper_unit_for_byte_size(consumed_by_advance_promotion), - byte_size_in_proper_unit(regular_promotion), proper_unit_for_byte_size(regular_promotion), + byte_size_in_proper_unit(regular_promotion), + proper_unit_for_byte_size(regular_promotion), byte_size_in_proper_unit(old_bytes_loaned_for_young_evac), proper_unit_for_byte_size(old_bytes_loaned_for_young_evac), - byte_size_in_proper_unit(allocation_supplement), proper_unit_for_byte_size(allocation_supplement), - byte_size_in_proper_unit(excess), proper_unit_for_byte_size(excess)); + byte_size_in_proper_unit(allocation_supplement), + proper_unit_for_byte_size(allocation_supplement), + byte_size_in_proper_unit(excess), + proper_unit_for_byte_size(excess)); } void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index e63edd667df0a..c3bdc924a9d1c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -63,15 +63,21 @@ class ShenandoahGeneration : public CHeapObj { private: // Compute evacuation budgets prior to choosing collection set. - void compute_evacuation_budgets(ShenandoahHeap* heap, bool* preselected_regions, ShenandoahCollectionSet* collection_set, - size_t &consumed_by_advance_promotion); + void compute_evacuation_budgets(ShenandoahHeap* heap, + bool* preselected_regions, + ShenandoahCollectionSet* collection_set, + size_t& consumed_by_advance_promotion); // Adjust evacuation budgets after choosing collection set. - void adjust_evacuation_budgets(ShenandoahHeap* heap, ShenandoahCollectionSet* collection_set, + void adjust_evacuation_budgets(ShenandoahHeap* heap, + ShenandoahCollectionSet* collection_set, size_t consumed_by_advance_promotion); public: - ShenandoahGeneration(ShenandoahGenerationType type, uint max_workers, size_t max_capacity, size_t soft_max_capacity); + ShenandoahGeneration(ShenandoahGenerationType type, + uint max_workers, + size_t max_capacity, + size_t soft_max_capacity); ~ShenandoahGeneration(); bool is_young() const { return _type == YOUNG; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp index e6cd04835b2b8..c7d6d8a6627f0 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp @@ -45,17 +45,20 @@ class ShenandoahGlobalGeneration : public ShenandoahGeneration { virtual size_t used() const override; virtual size_t available() const override; - virtual void set_concurrent_mark_in_progress(bool in_progress) override; + virtual void set_concurrent_mark_in_progress(bool in_progress) override; - bool contains(ShenandoahHeapRegion* region) const override; + bool contains(ShenandoahHeapRegion* region) const override; - bool contains(oop obj) const override { return true; } + bool contains(oop obj) const override { + // TODO: Should this assert is_in()? + return true; + } - void parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl) override; + void parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl) override; - void heap_region_iterate(ShenandoahHeapRegionClosure* cl) override; + void heap_region_iterate(ShenandoahHeapRegionClosure* cl) override; - bool is_concurrent_mark_in_progress() override; + bool is_concurrent_mark_in_progress() override; }; #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHGLOBALGENERATION_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 77fac13ee3d3d..ca7cfe3ee3a95 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -809,7 +809,7 @@ void ShenandoahHeap::op_uncommit(double shrink_before, size_t shrink_until) { void ShenandoahHeap::handle_old_evacuation(HeapWord* obj, size_t words, bool promotion) { // Only register the copy of the object that won the evacuation race. - card_scan()->register_object_wo_lock(obj); + card_scan()->register_object_without_lock(obj); // Mark the entire range of the evacuated object as dirty. At next remembered set scan, // we will clear dirty bits that do not hold interesting pointers. It's more efficient to @@ -1042,7 +1042,7 @@ void ShenandoahHeap::retire_plab(PLAB* plab, Thread* thread) { // safely walk the region backing the plab. log_debug(gc)("retire_plab() is registering remnant of size " SIZE_FORMAT " at " PTR_FORMAT, plab->waste() - waste, p2i(top)); - card_scan()->register_object_wo_lock(top); + card_scan()->register_object_without_lock(top); } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp index 696fb11b21b10..731b49f4cf6cc 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp @@ -624,7 +624,6 @@ inline void ShenandoahHeap::assert_lock_for_affiliation(ShenandoahAffiliation or // Note: during full GC, all transitions between states are possible. During Full GC, we should be in a safepoint. if ((orig_affiliation == ShenandoahAffiliation::FREE) || (new_affiliation == ShenandoahAffiliation::FREE)) { - extern bool _is_at_shenandoah_safepoint(); shenandoah_assert_heaplocked_or_fullgc_safepoint(); } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index ca59f87af001f..1385ffca97f7f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -435,7 +435,7 @@ void ShenandoahHeapRegion::print_on(outputStream* st) const { } // oop_iterate without closure and without cancellation. always return true. -bool ShenandoahHeapRegion::oop_fill_and_coalesce_wo_cancel() { +bool ShenandoahHeapRegion::oop_fill_and_coalesce_without_cancel() { HeapWord* obj_addr = resume_coalesce_and_fill(); assert(!is_humongous(), "No need to fill or coalesce humongous regions"); @@ -1059,7 +1059,7 @@ size_t ShenandoahHeapRegion::promote_humongous() { // Since this region may have served previously as OLD, it may hold obsolete object range info. heap->card_scan()->reset_object_range(bottom(), bottom() + spanned_regions * ShenandoahHeapRegion::region_size_words()); // Since the humongous region holds only one object, no lock is necessary for this register_object() invocation. - heap->card_scan()->register_object_wo_lock(bottom()); + heap->card_scan()->register_object_without_lock(bottom()); if (obj->is_typeArray()) { // Primitive arrays don't need to be scanned. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp index 20c9340699527..a422f7e25c5d0 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp @@ -397,7 +397,7 @@ class ShenandoahHeapRegion { bool oop_fill_and_coalesce(); // Like oop_fill_and_coalesce(), but without honoring cancellation requests. - bool oop_fill_and_coalesce_wo_cancel(); + bool oop_fill_and_coalesce_without_cancel(); // During global collections, this service iterates through an old-gen heap region that is not part of collection // set to fill and register ranges of dead memory. Note that live objects were previously registered. Some dead objects diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp index 16b794c2c700e..0c719d4abfcde 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp @@ -619,12 +619,12 @@ class ShenandoahCardCluster: public CHeapObj { // before calling it. void register_object(HeapWord* address); - // register_object_wo_lock() does not require that the caller hold + // register_object_without_lock() does not require that the caller hold // the heap lock before calling it, under the assumption that the // caller has assure no other thread will endeavor to concurrently // register objects that start within the same card's memory region // as address. - void register_object_wo_lock(HeapWord* address); + void register_object_without_lock(HeapWord* address); // During the reference updates phase of GC, we walk through each old-gen memory region that was // not part of the collection set and we invalidate all unmarked objects. As part of this effort, @@ -840,7 +840,7 @@ class ShenandoahScanRemembered: public CHeapObj { void reset_object_range(HeapWord *from, HeapWord *to); void register_object(HeapWord *addr); - void register_object_wo_lock(HeapWord *addr); + void register_object_without_lock(HeapWord *addr); void coalesce_objects(HeapWord *addr, size_t length_in_words); HeapWord* first_object_in_card(size_t card_index) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp index 4a3fcb69acb9d..465c6b8fb392b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp @@ -183,12 +183,12 @@ inline void ShenandoahCardCluster::register_object(HeapWord* address) { shenandoah_assert_heaplocked(); - register_object_wo_lock(address); + register_object_without_lock(address); } template inline void -ShenandoahCardCluster::register_object_wo_lock(HeapWord* address) { +ShenandoahCardCluster::register_object_without_lock(HeapWord* address) { size_t card_at_start = _rs->card_index_for_addr(address); HeapWord *card_start_address = _rs->addr_for_card_index(card_at_start); uint8_t offset_in_card = address - card_start_address; @@ -419,8 +419,8 @@ ShenandoahScanRemembered::register_object(HeapWord *addr) { template inline void -ShenandoahScanRemembered::register_object_wo_lock(HeapWord *addr) { - _scc->register_object_wo_lock(addr); +ShenandoahScanRemembered::register_object_without_lock(HeapWord *addr) { + _scc->register_object_without_lock(addr); } template From 9d87724697428f4c75cd04a062df35cddf5a56c6 Mon Sep 17 00:00:00 2001 From: Aleksey Shipilev Date: Thu, 13 Apr 2023 08:27:38 +0000 Subject: [PATCH 212/254] Restore event descriptions immortality Reviewed-by: kdnilsen, ysr --- .../gc/shenandoah/shenandoahConcurrentGC.cpp | 28 +++++++++---------- .../gc/shenandoah/shenandoahConcurrentGC.hpp | 9 +++--- .../gc/shenandoah/shenandoahDegeneratedGC.cpp | 24 +++++++++++++--- .../gc/shenandoah/shenandoahDegeneratedGC.hpp | 3 +- .../share/gc/shenandoah/shenandoahUtils.hpp | 16 +++++++++++ 5 files changed, 56 insertions(+), 24 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index 3a60cdd3798f3..85b0a5c1d0a34 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -295,8 +295,7 @@ void ShenandoahConcurrentGC::vmop_entry_final_roots(bool increment_region_ages) } void ShenandoahConcurrentGC::entry_init_mark() { - char msg[1024]; - init_mark_event_message(msg, sizeof(msg)); + const char* msg = init_mark_event_message(); ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::init_mark); EventMark em("%s", msg); @@ -308,8 +307,7 @@ void ShenandoahConcurrentGC::entry_init_mark() { } void ShenandoahConcurrentGC::entry_final_mark() { - char msg[1024]; - final_mark_event_message(msg, sizeof(msg)); + const char* msg = final_mark_event_message(); ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::final_mark); EventMark em("%s", msg); @@ -397,10 +395,9 @@ void ShenandoahConcurrentGC::entry_mark_roots() { } void ShenandoahConcurrentGC::entry_mark() { - char msg[1024]; ShenandoahHeap* const heap = ShenandoahHeap::heap(); TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); - conc_mark_event_message(msg, sizeof(msg)); + const char* msg = conc_mark_event_message(); ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::conc_mark); EventMark em("%s", msg); @@ -1226,34 +1223,35 @@ bool ShenandoahConcurrentGC::check_cancellation_and_abort(ShenandoahDegenPoint p return false; } -void ShenandoahConcurrentGC::init_mark_event_message(char* buf, size_t len) const { +const char* ShenandoahConcurrentGC::init_mark_event_message() const { ShenandoahHeap* const heap = ShenandoahHeap::heap(); assert(!heap->has_forwarded_objects(), "Should not have forwarded objects here"); if (heap->unload_classes()) { - jio_snprintf(buf, len, "Pause Init Mark (%s) (unload classes)", _generation->name()); + SHENANDOAH_RETURN_EVENT_MESSAGE(heap, _generation->type(), "Pause Init Mark", " (unload classes)"); } else { - jio_snprintf(buf, len, "Pause Init Mark (%s)", _generation->name()); + SHENANDOAH_RETURN_EVENT_MESSAGE(heap, _generation->type(), "Pause Init Mark", ""); } } -void ShenandoahConcurrentGC::final_mark_event_message(char* buf, size_t len) const { +const char* ShenandoahConcurrentGC::final_mark_event_message() const { ShenandoahHeap* const heap = ShenandoahHeap::heap(); assert(!heap->has_forwarded_objects() || heap->is_concurrent_old_mark_in_progress(), "Should not have forwarded objects during final mark, unless old gen concurrent mark is running"); + if (heap->unload_classes()) { - jio_snprintf(buf, len, "Pause Final Mark (%s) (unload classes)", _generation->name()); + SHENANDOAH_RETURN_EVENT_MESSAGE(heap, _generation->type(), "Pause Final Mark", " (unload classes)"); } else { - jio_snprintf(buf, len, "Pause Final Mark (%s)", _generation->name()); + SHENANDOAH_RETURN_EVENT_MESSAGE(heap, _generation->type(), "Pause Final Mark", ""); } } -void ShenandoahConcurrentGC::conc_mark_event_message(char* buf, size_t len) const { +const char* ShenandoahConcurrentGC::conc_mark_event_message() const { ShenandoahHeap* const heap = ShenandoahHeap::heap(); assert(!heap->has_forwarded_objects() || heap->is_concurrent_old_mark_in_progress(), "Should not have forwarded objects concurrent mark, unless old gen concurrent mark is running"); if (heap->unload_classes()) { - jio_snprintf(buf, len, "Concurrent marking (%s) (unload classes)", _generation->name()); + SHENANDOAH_RETURN_EVENT_MESSAGE(heap, _generation->type(), "Concurrent marking", " (unload classes)"); } else { - jio_snprintf(buf, len, "Concurrent marking (%s)", _generation->name()); + SHENANDOAH_RETURN_EVENT_MESSAGE(heap, _generation->type(), "Concurrent marking", ""); } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp index c7d64b70fef60..7d0fb0ecbd969 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp @@ -130,10 +130,11 @@ class ShenandoahConcurrentGC : public ShenandoahGC { private: void start_mark(); - // Messages for GC trace events - void init_mark_event_message(char* buf, size_t len) const; - void final_mark_event_message(char* buf, size_t len) const; - void conc_mark_event_message(char* buf, size_t len) const; + // Messages for GC trace events, they have to be immortal for + // passing around the logging/tracing systems + const char* init_mark_event_message() const; + const char* final_mark_event_message() const; + const char* conc_mark_event_message() const; protected: // Check GC cancellation and abort concurrent GC diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp index ef288048f4ee4..fa62f7a9fe8e9 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp @@ -70,8 +70,7 @@ void ShenandoahDegenGC::vmop_degenerated() { } void ShenandoahDegenGC::entry_degenerated() { - char msg[1024]; - degen_event_message(_degen_point, msg, sizeof(msg)); + const char* msg = degen_event_message(_degen_point); ShenandoahPausePhase gc_phase(msg, ShenandoahPhaseTimings::degen_gc, true /* log_heap_usage */); EventMark em("%s", msg); ShenandoahHeap* const heap = ShenandoahHeap::heap(); @@ -442,8 +441,25 @@ void ShenandoahDegenGC::op_degenerated_futile() { full_gc.op_full(GCCause::_shenandoah_upgrade_to_full_gc); } -void ShenandoahDegenGC::degen_event_message(ShenandoahDegenPoint point, char* buf, size_t len) const { - jio_snprintf(buf, len, "Pause Degenerated %s GC (%s)", _generation->name(), ShenandoahGC::degen_point_to_string(point)); +const char* ShenandoahDegenGC::degen_event_message(ShenandoahDegenPoint point) const { + const ShenandoahHeap* heap = ShenandoahHeap::heap(); + switch (point) { + case _degenerated_unset: + SHENANDOAH_RETURN_EVENT_MESSAGE(heap, _generation->type(), "Pause Degenerated GC", " ()"); + case _degenerated_outside_cycle: + SHENANDOAH_RETURN_EVENT_MESSAGE(heap, _generation->type(), "Pause Degenerated GC", " (Outside of Cycle)"); + case _degenerated_roots: + SHENANDOAH_RETURN_EVENT_MESSAGE(heap, _generation->type(), "Pause Degenerated GC", " (Roots)"); + case _degenerated_mark: + SHENANDOAH_RETURN_EVENT_MESSAGE(heap, _generation->type(), "Pause Degenerated GC", " (Mark)"); + case _degenerated_evac: + SHENANDOAH_RETURN_EVENT_MESSAGE(heap, _generation->type(), "Pause Degenerated GC", " (Evacuation)"); + case _degenerated_updaterefs: + SHENANDOAH_RETURN_EVENT_MESSAGE(heap, _generation->type(), "Pause Degenerated GC", " (Update Refs)"); + default: + ShouldNotReachHere(); + SHENANDOAH_RETURN_EVENT_MESSAGE(heap, _generation->type(), "Pause Degenerated GC", " (?)"); + } } void ShenandoahDegenGC::upgrade_to_full() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.hpp index 9e92e5b2aa07d..32d26661ad076 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.hpp @@ -64,8 +64,9 @@ class ShenandoahDegenGC : public ShenandoahGC { void op_degenerated_futile(); void op_degenerated_fail(); - void degen_event_message(ShenandoahDegenPoint point, char* buf, size_t len) const; void upgrade_to_full(); + + const char* degen_event_message(ShenandoahDegenPoint point) const; }; #endif // SHARE_GC_SHENANDOAH_SHENANDOAHDEGENERATEDGC_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp b/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp index 6dac5e46e2a32..c8cdbe2f96341 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp @@ -44,6 +44,22 @@ class GCTimer; class ShenandoahGeneration; +#define SHENANDOAH_RETURN_EVENT_MESSAGE(heap, generation_type, prefix, postfix) \ + if (!heap->mode()->is_generational()) { \ + return prefix "" postfix; \ + } \ + switch (generation_type) { \ + case GLOBAL: \ + return prefix " (GLOBAL)" postfix; \ + case YOUNG: \ + return prefix " (YOUNG)" postfix; \ + case OLD: \ + return prefix " (OLD)" postfix; \ + default: \ + ShouldNotReachHere(); \ + return prefix " (?)" postfix; \ + } \ + class ShenandoahGCSession : public StackObj { private: ShenandoahHeap* const _heap; From 9f256a5ac31e943b6c3781e36559b812c4dd6201 Mon Sep 17 00:00:00 2001 From: Aleksey Shipilev Date: Thu, 13 Apr 2023 08:33:35 +0000 Subject: [PATCH 213/254] Cleanups, TODOs, asserts (part 2) Reviewed-by: ysr, kdnilsen --- .../share/gc/shenandoah/shenandoahFullGC.cpp | 6 +- .../share/gc/shenandoah/shenandoahHeap.cpp | 167 --------- .../share/gc/shenandoah/shenandoahHeap.hpp | 6 - .../share/gc/shenandoah/shenandoahMark.cpp | 10 +- .../share/gc/shenandoah/shenandoahMark.hpp | 2 +- .../gc/shenandoah/shenandoahMark.inline.hpp | 26 +- .../shenandoah/shenandoahMarkingContext.cpp | 28 +- .../shenandoah/shenandoahMarkingContext.hpp | 1 + .../shenandoahMarkingContext.inline.hpp | 65 ++-- .../gc/shenandoah/shenandoahMemoryPool.hpp | 2 +- .../gc/shenandoah/shenandoahOldGeneration.cpp | 104 +++--- .../gc/shenandoah/shenandoahOldGeneration.hpp | 53 ++- .../gc/shenandoah/shenandoahOopClosures.hpp | 43 +-- .../shenandoahOopClosures.inline.hpp | 18 - .../gc/shenandoah/shenandoahPhaseTimings.hpp | 8 +- .../share/gc/shenandoah/shenandoahSTWMark.cpp | 12 +- .../shenandoah/shenandoahThreadLocalData.hpp | 3 +- .../share/gc/shenandoah/shenandoahUnload.cpp | 1 + .../gc/shenandoah/shenandoahVMOperations.cpp | 3 +- .../gc/shenandoah/shenandoahVMOperations.hpp | 9 +- .../gc/shenandoah/shenandoahVerifier.cpp | 321 +++++++++++++++--- .../gc/shenandoah/shenandoahVerifier.hpp | 29 +- .../shenandoah/shenandoahYoungGeneration.cpp | 4 - .../shenandoah/shenandoahYoungGeneration.hpp | 19 +- 24 files changed, 478 insertions(+), 462 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index d2aa7a1fcaede..2da28e5534ed0 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -355,11 +355,7 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) { heap->set_full_gc_in_progress(false); if (ShenandoahVerify) { - if (heap->mode()->is_generational()) { - heap->verifier()->verify_after_generational_fullgc(); - } else { - heap->verifier()->verify_after_fullgc(); - } + heap->verifier()->verify_after_fullgc(); } // Having reclaimed all dead memory, it is now safe to restore capacities to original values. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index ca7cfe3ee3a95..57a14962ed068 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -3132,173 +3132,6 @@ void ShenandoahGenerationRegionClosure::heap_region_do(ShenandoahHeapReg _cl->heap_region_do(region); } -// Assure that the remember set has a dirty card everywhere there is an interesting pointer. -// This examines the read_card_table between bottom() and top() since all PLABS are retired -// before the safepoint for init_mark. Actually, we retire them before update-references and don't -// restore them until the start of evacuation. -void ShenandoahHeap::verify_rem_set_at_mark() { - shenandoah_assert_safepoint(); - assert(mode()->is_generational(), "Only verify remembered set for generational operational modes"); - - ShenandoahRegionIterator iterator; - RememberedScanner* scanner = card_scan(); - ShenandoahVerifyRemSetClosure check_interesting_pointers(true); - ShenandoahMarkingContext* ctx; - - log_debug(gc)("Verifying remembered set at %s mark", doing_mixed_evacuations()? "mixed": "young"); - - if (is_old_bitmap_stable() || active_generation()->is_global()) { - ctx = complete_marking_context(); - } else { - ctx = nullptr; - } - - while (iterator.has_next()) { - ShenandoahHeapRegion* r = iterator.next(); - HeapWord* tams = ctx? ctx->top_at_mark_start(r): nullptr; - if (r == nullptr) - break; - if (r->is_old() && r->is_active()) { - HeapWord* obj_addr = r->bottom(); - if (r->is_humongous_start()) { - oop obj = cast_to_oop(obj_addr); - if (!ctx || ctx->is_marked(obj)) { - // For humongous objects, the typical object is an array, so the following checks may be overkill - // For regular objects (not object arrays), if the card holding the start of the object is dirty, - // we do not need to verify that cards spanning interesting pointers within this object are dirty. - if (!scanner->is_card_dirty(obj_addr) || obj->is_objArray()) { - obj->oop_iterate(&check_interesting_pointers); - } - // else, object's start is marked dirty and obj is not an objArray, so any interesting pointers are covered - } - // else, this humongous object is not marked so no need to verify its internal pointers - if (!scanner->verify_registration(obj_addr, ctx)) { - ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, obj_addr, nullptr, - "Verify init-mark remembered set violation", "object not properly registered", __FILE__, __LINE__); - } - } else if (!r->is_humongous()) { - HeapWord* top = r->top(); - while (obj_addr < top) { - oop obj = cast_to_oop(obj_addr); - // ctx->is_marked() returns true if mark bit set (TAMS not relevant during init mark) - if (!ctx || ctx->is_marked(obj)) { - // For regular objects (not object arrays), if the card holding the start of the object is dirty, - // we do not need to verify that cards spanning interesting pointers within this object are dirty. - if (!scanner->is_card_dirty(obj_addr) || obj->is_objArray()) { - obj->oop_iterate(&check_interesting_pointers); - } - // else, object's start is marked dirty and obj is not an objArray, so any interesting pointers are covered - if (!scanner->verify_registration(obj_addr, ctx)) { - ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, obj_addr, nullptr, - "Verify init-mark remembered set violation", "object not properly registered", __FILE__, __LINE__); - } - obj_addr += obj->size(); - } else { - // This object is not live so we don't verify dirty cards contained therein - assert(tams != nullptr, "If object is not live, ctx and tams should be non-null"); - obj_addr = ctx->get_next_marked_addr(obj_addr, tams); - } - } - } // else, we ignore humongous continuation region - } // else, this is not an OLD region so we ignore it - } // all regions have been processed -} - -void ShenandoahHeap::help_verify_region_rem_set(ShenandoahHeapRegion* r, ShenandoahMarkingContext* ctx, HeapWord* from, - HeapWord* top, HeapWord* registration_watermark, const char* message) { - RememberedScanner* scanner = card_scan(); - ShenandoahVerifyRemSetClosure check_interesting_pointers(false); - - HeapWord* obj_addr = from; - if (r->is_humongous_start()) { - oop obj = cast_to_oop(obj_addr); - if (!ctx || ctx->is_marked(obj)) { - size_t card_index = scanner->card_index_for_addr(obj_addr); - // For humongous objects, the typical object is an array, so the following checks may be overkill - // For regular objects (not object arrays), if the card holding the start of the object is dirty, - // we do not need to verify that cards spanning interesting pointers within this object are dirty. - if (!scanner->is_write_card_dirty(card_index) || obj->is_objArray()) { - obj->oop_iterate(&check_interesting_pointers); - } - // else, object's start is marked dirty and obj is not an objArray, so any interesting pointers are covered - } - // else, this humongous object is not live so no need to verify its internal pointers - - if ((obj_addr < registration_watermark) && !scanner->verify_registration(obj_addr, ctx)) { - ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, obj_addr, nullptr, message, - "object not properly registered", __FILE__, __LINE__); - } - } else if (!r->is_humongous()) { - while (obj_addr < top) { - oop obj = cast_to_oop(obj_addr); - // ctx->is_marked() returns true if mark bit set or if obj above TAMS. - if (!ctx || ctx->is_marked(obj)) { - size_t card_index = scanner->card_index_for_addr(obj_addr); - // For regular objects (not object arrays), if the card holding the start of the object is dirty, - // we do not need to verify that cards spanning interesting pointers within this object are dirty. - if (!scanner->is_write_card_dirty(card_index) || obj->is_objArray()) { - obj->oop_iterate(&check_interesting_pointers); - } - // else, object's start is marked dirty and obj is not an objArray, so any interesting pointers are covered - - if ((obj_addr < registration_watermark) && !scanner->verify_registration(obj_addr, ctx)) { - ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, obj_addr, nullptr, message, - "object not properly registered", __FILE__, __LINE__); - } - obj_addr += obj->size(); - } else { - // This object is not live so we don't verify dirty cards contained therein - HeapWord* tams = ctx->top_at_mark_start(r); - obj_addr = ctx->get_next_marked_addr(obj_addr, tams); - } - } - } -} - -void ShenandoahHeap::verify_rem_set_after_full_gc() { - shenandoah_assert_safepoint(); - assert(mode()->is_generational(), "Only verify remembered set for generational operational modes"); - - ShenandoahRegionIterator iterator; - - while (iterator.has_next()) { - ShenandoahHeapRegion* r = iterator.next(); - if (r == nullptr) - break; - if (r->is_old() && !r->is_cset()) { - help_verify_region_rem_set(r, nullptr, r->bottom(), r->top(), r->top(), "Remembered set violation at end of Full GC"); - } - } -} - -// Assure that the remember set has a dirty card everywhere there is an interesting pointer. Even though -// the update-references scan of remembered set only examines cards up to update_watermark, the remembered -// set should be valid through top. This examines the write_card_table between bottom() and top() because -// all PLABS are retired immediately before the start of update refs. -void ShenandoahHeap::verify_rem_set_at_update_ref() { - shenandoah_assert_safepoint(); - assert(mode()->is_generational(), "Only verify remembered set for generational operational modes"); - - ShenandoahRegionIterator iterator; - ShenandoahMarkingContext* ctx; - - if (is_old_bitmap_stable() || active_generation()->is_global()) { - ctx = complete_marking_context(); - } else { - ctx = nullptr; - } - - while (iterator.has_next()) { - ShenandoahHeapRegion* r = iterator.next(); - if (r == nullptr) - break; - if (r->is_old() && !r->is_cset()) { - help_verify_region_rem_set(r, ctx, r->bottom(), r->top(), r->get_update_watermark(), - "Remembered set violation at init-update-references"); - } - } -} - ShenandoahGeneration* ShenandoahHeap::generation_for(ShenandoahAffiliation affiliation) const { if (!mode()->is_generational()) { return global_generation(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index 0fe5db7923006..31f1c0a08798e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -206,9 +206,6 @@ class ShenandoahHeap : public CollectedHeap { void prepare_for_verify() override; void verify(VerifyOption vo) override; - void verify_rem_set_at_mark(); - void verify_rem_set_at_update_ref(); - void verify_rem_set_after_full_gc(); // WhiteBox testing support. bool supports_concurrent_gc_breakpoints() const override { @@ -226,9 +223,6 @@ class ShenandoahHeap : public CollectedHeap { volatile size_t _committed; shenandoah_padding(1); - void help_verify_region_rem_set(ShenandoahHeapRegion* r, ShenandoahMarkingContext* ctx, - HeapWord* from, HeapWord* top, HeapWord* update_watermark, const char* message); - public: void increase_used(size_t bytes); void decrease_used(size_t bytes); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp index dd013fcfcb379..7cf3c3ed9f163 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp @@ -67,7 +67,7 @@ ShenandoahMark::ShenandoahMark(ShenandoahGeneration* generation) : template void ShenandoahMark::mark_loop_prework(uint w, TaskTerminator *t, ShenandoahReferenceProcessor *rp, StringDedup::Requests* const req, bool update_refs) { ShenandoahObjToScanQueue* q = get_queue(w); - ShenandoahObjToScanQueue* old = get_old_queue(w); + ShenandoahObjToScanQueue* old_q = get_old_queue(w); ShenandoahHeap* const heap = ShenandoahHeap::heap(); ShenandoahLiveData* ld = heap->get_liveness_cache(w); @@ -76,11 +76,11 @@ void ShenandoahMark::mark_loop_prework(uint w, TaskTerminator *t, ShenandoahRefe // play nice with specialized_oop_iterators. if (update_refs) { using Closure = ShenandoahMarkUpdateRefsClosure; - Closure cl(q, rp, old); + Closure cl(q, rp, old_q); mark_loop_work(&cl, ld, w, t, req); } else { using Closure = ShenandoahMarkRefsClosure; - Closure cl(q, rp, old); + Closure cl(q, rp, old_q); mark_loop_work(&cl, ld, w, t, req); } @@ -175,9 +175,9 @@ void ShenandoahMark::mark_loop_work(T* cl, ShenandoahLiveData* live_data, uint w } } q = get_queue(worker_id); - ShenandoahObjToScanQueue* old = get_old_queue(worker_id); + ShenandoahObjToScanQueue* old_q = get_old_queue(worker_id); - ShenandoahSATBBufferClosure drain_satb(q, old); + ShenandoahSATBBufferClosure drain_satb(q, old_q); SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set(); /* diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.hpp index cbdc767315422..b36d57751b992 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMark.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.hpp @@ -46,7 +46,7 @@ class ShenandoahMark: public StackObj { public: template - static inline void mark_through_ref(T* p, ShenandoahObjToScanQueue* q, ShenandoahObjToScanQueue* old, ShenandoahMarkingContext* const mark_context, bool weak); + static inline void mark_through_ref(T* p, ShenandoahObjToScanQueue* q, ShenandoahObjToScanQueue* old_q, ShenandoahMarkingContext* const mark_context, bool weak); // Loom support void start_mark(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp index 1610936d1c8ae..917a696a673fb 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp @@ -241,13 +241,13 @@ template class ShenandoahSATBBufferClosure : public SATBBufferClosure { private: ShenandoahObjToScanQueue* _queue; - ShenandoahObjToScanQueue* _old; + ShenandoahObjToScanQueue* _old_queue; ShenandoahHeap* _heap; ShenandoahMarkingContext* const _mark_context; public: - ShenandoahSATBBufferClosure(ShenandoahObjToScanQueue* q, ShenandoahObjToScanQueue* old) : + ShenandoahSATBBufferClosure(ShenandoahObjToScanQueue* q, ShenandoahObjToScanQueue* old_q) : _queue(q), - _old(old), + _old_queue(old_q), _heap(ShenandoahHeap::heap()), _mark_context(_heap->marking_context()) { @@ -257,7 +257,7 @@ class ShenandoahSATBBufferClosure : public SATBBufferClosure { assert(size == 0 || !_heap->has_forwarded_objects() || _heap->is_concurrent_old_mark_in_progress(), "Forwarded objects are not expected here"); for (size_t i = 0; i < size; ++i) { oop *p = (oop *) &buffer[i]; - ShenandoahMark::mark_through_ref(p, _queue, _old, _mark_context, false); + ShenandoahMark::mark_through_ref(p, _queue, _old_queue, _mark_context, false); } } }; @@ -265,18 +265,19 @@ class ShenandoahSATBBufferClosure : public SATBBufferClosure { template bool ShenandoahMark::in_generation(oop obj) { // Each in-line expansion of in_generation() resolves GENERATION at compile time. - if (GENERATION == YOUNG) + if (GENERATION == YOUNG) { return ShenandoahHeap::heap()->is_in_young(obj); - else if (GENERATION == OLD) + } else if (GENERATION == OLD) { return ShenandoahHeap::heap()->is_in_old(obj); - else if (GENERATION == GLOBAL) + } else if (GENERATION == GLOBAL) { return true; - else + } else { return false; + } } template -inline void ShenandoahMark::mark_through_ref(T *p, ShenandoahObjToScanQueue* q, ShenandoahObjToScanQueue* old, ShenandoahMarkingContext* const mark_context, bool weak) { +inline void ShenandoahMark::mark_through_ref(T *p, ShenandoahObjToScanQueue* q, ShenandoahObjToScanQueue* old_q, ShenandoahMarkingContext* const mark_context, bool weak) { T o = RawAccess<>::oop_load(p); if (!CompressedOops::is_null(o)) { oop obj = CompressedOops::decode_not_null(o); @@ -287,6 +288,7 @@ inline void ShenandoahMark::mark_through_ref(T *p, ShenandoahObjToScanQueue* q, if (in_generation(obj)) { mark_ref(q, mark_context, weak, obj); shenandoah_assert_marked(p, obj); + // TODO: This is v-call on very hot path, can we sense the same from GENERATION? if (heap->mode()->is_generational()) { // TODO: As implemented herein, GLOBAL collections reconstruct the card table during GLOBAL concurrent // marking. Note that the card table is cleaned at init_mark time so it needs to be reconstructed to support @@ -301,9 +303,9 @@ inline void ShenandoahMark::mark_through_ref(T *p, ShenandoahObjToScanQueue* q, heap->mark_card_as_dirty((HeapWord*)p); } } - } else if (old != nullptr) { - // Young mark, bootstrapping old or concurrent with old marking. - mark_ref(old, mark_context, weak, obj); + } else if (old_q != nullptr) { + // Young mark, bootstrapping old_q or concurrent with old_q marking. + mark_ref(old_q, mark_context, weak, obj); shenandoah_assert_marked(p, obj); } else if (GENERATION == OLD) { // Old mark, found a young pointer. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.cpp index 81abb2f4e5817..e031cc7c82c70 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.cpp @@ -55,8 +55,9 @@ bool ShenandoahMarkingContext::is_bitmap_clear_range(const HeapWord* start, cons size_t end_idx = heap->heap_region_index_containing(end - 1); while (start_idx <= end_idx) { ShenandoahHeapRegion* r = heap->get_region(start_idx); - if (!heap->is_bitmap_slice_committed(r)) + if (!heap->is_bitmap_slice_committed(r)) { return true; + } start_idx++; } } @@ -79,23 +80,28 @@ HeapWord* ShenandoahMarkingContext::top_bitmap(ShenandoahHeapRegion* r) { } void ShenandoahMarkingContext::clear_bitmap(ShenandoahHeapRegion* r) { + if (!r->is_affiliated()) { + // Heap iterators include FREE regions, which don't need to be cleared. + // TODO: would be better for certain iterators to not include FREE regions. + return; + } + HeapWord* bottom = r->bottom(); HeapWord* top_bitmap = _top_bitmaps[r->index()]; log_debug(gc)("SMC:clear_bitmap for %s Region " SIZE_FORMAT ", top_bitmap: " PTR_FORMAT, r->affiliation_name(), r->index(), p2i(top_bitmap)); - if (r->is_affiliated()) { - if (top_bitmap > bottom) { - _mark_bit_map.clear_range_large(MemRegion(bottom, top_bitmap)); - _top_bitmaps[r->index()] = bottom; - } - r->clear_live_data(); - assert(is_bitmap_clear_range(bottom, r->end()), - "Region " SIZE_FORMAT " should have no marks in bitmap", r->index()); + if (top_bitmap > bottom) { + _mark_bit_map.clear_range_large(MemRegion(bottom, top_bitmap)); + _top_bitmaps[r->index()] = bottom; } - // heap iterators include FREE regions, which don't need to be cleared. - // TODO: would be better for certain iterators to not include FREE regions. + + // TODO: Why is clear_live_data here? + r->clear_live_data(); + + assert(is_bitmap_clear_range(bottom, r->end()), + "Region " SIZE_FORMAT " should have no marks in bitmap", r->index()); } bool ShenandoahMarkingContext::is_complete() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.hpp index 3b73384741553..1a77a0beb00f9 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.hpp @@ -60,6 +60,7 @@ class ShenandoahMarkingContext : public CHeapObj { inline bool mark_weak(oop obj); // Simple versions of marking accessors, to be used outside of marking (e.g. no possible concurrent updates) + // TODO: Do these really need to be const? inline bool is_marked(const oop) const; inline bool is_marked_strong(const oop obj) const; inline bool is_marked_weak(const oop obj) const; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.inline.hpp index cee8e1574600d..5f039535404b3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMarkingContext.inline.hpp @@ -75,38 +75,41 @@ inline bool ShenandoahMarkingContext::allocated_after_mark_start(const HeapWord* } inline void ShenandoahMarkingContext::capture_top_at_mark_start(ShenandoahHeapRegion *r) { - if (r->is_affiliated()) { - size_t idx = r->index(); - HeapWord* old_tams = _top_at_mark_starts_base[idx]; - HeapWord* new_tams = r->top(); - - assert(new_tams >= old_tams, - "Region " SIZE_FORMAT", TAMS updates should be monotonic: " PTR_FORMAT " -> " PTR_FORMAT, - idx, p2i(old_tams), p2i(new_tams)); - assert((new_tams == r->bottom()) || (old_tams == r->bottom()) || (new_tams >= _top_bitmaps[idx]), - "Region " SIZE_FORMAT", top_bitmaps updates should be monotonic: " PTR_FORMAT " -> " PTR_FORMAT, - idx, p2i(_top_bitmaps[idx]), p2i(new_tams)); - assert(old_tams == r->bottom() || is_bitmap_clear_range(old_tams, new_tams), - "Region " SIZE_FORMAT ", bitmap should be clear while adjusting TAMS: " PTR_FORMAT " -> " PTR_FORMAT, - idx, p2i(old_tams), p2i(new_tams)); - - log_debug(gc)("Capturing TAMS for %s Region " SIZE_FORMAT ", was: %llx, now: %llx", - r->affiliation_name(), idx, (unsigned long long) old_tams, (unsigned long long) new_tams); - - if ((old_tams == r->bottom()) && (new_tams > old_tams)) { - log_debug(gc)("Clearing mark bitmap for %s Region " SIZE_FORMAT " while capturing TAMS", - r->affiliation_name(), idx); - - clear_bitmap(r); - } - - _top_at_mark_starts_base[idx] = new_tams; - if (new_tams > r->bottom()) { - // In this case, new_tams is greater than old _top_bitmaps[idx] - _top_bitmaps[idx] = new_tams; - } + if (!r->is_affiliated()) { + // Non-affiliated regions do not need their TAMS updated + return; + } + + size_t idx = r->index(); + HeapWord* old_tams = _top_at_mark_starts_base[idx]; + HeapWord* new_tams = r->top(); + + assert(new_tams >= old_tams, + "Region " SIZE_FORMAT", TAMS updates should be monotonic: " PTR_FORMAT " -> " PTR_FORMAT, + idx, p2i(old_tams), p2i(new_tams)); + assert((new_tams == r->bottom()) || (old_tams == r->bottom()) || (new_tams >= _top_bitmaps[idx]), + "Region " SIZE_FORMAT", top_bitmaps updates should be monotonic: " PTR_FORMAT " -> " PTR_FORMAT, + idx, p2i(_top_bitmaps[idx]), p2i(new_tams)); + assert(old_tams == r->bottom() || is_bitmap_clear_range(old_tams, new_tams), + "Region " SIZE_FORMAT ", bitmap should be clear while adjusting TAMS: " PTR_FORMAT " -> " PTR_FORMAT, + idx, p2i(old_tams), p2i(new_tams)); + + log_debug(gc)("Capturing TAMS for %s Region " SIZE_FORMAT ", was: " PTR_FORMAT ", now: " PTR_FORMAT, + r->affiliation_name(), idx, p2i(old_tams), p2i(new_tams)); + + if ((old_tams == r->bottom()) && (new_tams > old_tams)) { + log_debug(gc)("Clearing mark bitmap for %s Region " SIZE_FORMAT " while capturing TAMS", + r->affiliation_name(), idx); + // TODO: Do we really need to do bitmap clears here? + // This could take a while, and we would instead like to clear bitmaps outside the pause. + clear_bitmap(r); + } + + _top_at_mark_starts_base[idx] = new_tams; + if (new_tams > r->bottom()) { + // In this case, new_tams is greater than old _top_bitmaps[idx] + _top_bitmaps[idx] = new_tams; } - // else, FREE regions do not need their TAMS updated } inline void ShenandoahMarkingContext::reset_top_at_mark_start(ShenandoahHeapRegion* r) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMemoryPool.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMemoryPool.hpp index 2743e65edd225..3cbbc0e9a8b6f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMemoryPool.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMemoryPool.hpp @@ -38,7 +38,7 @@ class ShenandoahMemoryPool : public CollectedMemoryPool { public: ShenandoahMemoryPool(ShenandoahHeap* pool, - const char* name = "Shenandoah"); + const char* name = "Shenandoah"); virtual MemoryUsage get_memory_usage(); virtual size_t used_in_bytes(); virtual size_t max_size() const; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp index b197e2bdf887b..291a129e10078 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp @@ -52,10 +52,10 @@ #include "utilities/events.hpp" class ShenandoahFlushAllSATB : public ThreadClosure { - private: +private: SATBMarkQueueSet& _satb_qset; - public: +public: explicit ShenandoahFlushAllSATB(SATBMarkQueueSet& satb_qset) : _satb_qset(satb_qset) {} @@ -66,21 +66,20 @@ class ShenandoahFlushAllSATB : public ThreadClosure { }; class ShenandoahProcessOldSATB : public SATBBufferClosure { - private: - ShenandoahObjToScanQueue* _queue; - ShenandoahHeap* _heap; +private: + ShenandoahObjToScanQueue* _queue; + ShenandoahHeap* _heap; ShenandoahMarkingContext* const _mark_context; + size_t _trashed_oops; - public: - size_t _trashed_oops; - +public: explicit ShenandoahProcessOldSATB(ShenandoahObjToScanQueue* q) : _queue(q), _heap(ShenandoahHeap::heap()), _mark_context(_heap->marking_context()), _trashed_oops(0) {} - void do_buffer(void **buffer, size_t size) { + void do_buffer(void** buffer, size_t size) { assert(size == 0 || !_heap->has_forwarded_objects() || _heap->is_concurrent_old_mark_in_progress(), "Forwarded objects are not expected here"); for (size_t i = 0; i < size; ++i) { oop *p = (oop *) &buffer[i]; @@ -88,19 +87,22 @@ class ShenandoahProcessOldSATB : public SATBBufferClosure { if (region->is_old() && region->is_active()) { ShenandoahMark::mark_through_ref(p, _queue, nullptr, _mark_context, false); } else { - ++_trashed_oops; + _trashed_oops++; } } } + + size_t trashed_oops() { + return _trashed_oops; + } }; class ShenandoahPurgeSATBTask : public WorkerTask { private: ShenandoahObjToScanQueueSet* _mark_queues; + volatile size_t _trashed_oops; public: - volatile size_t _trashed_oops; - explicit ShenandoahPurgeSATBTask(ShenandoahObjToScanQueueSet* queues) : WorkerTask("Purge SATB"), _mark_queues(queues), @@ -110,7 +112,7 @@ class ShenandoahPurgeSATBTask : public WorkerTask { ~ShenandoahPurgeSATBTask() { if (_trashed_oops > 0) { - log_info(gc)("Purged " SIZE_FORMAT " oops from old generation SATB buffers.", _trashed_oops); + log_info(gc)("Purged " SIZE_FORMAT " oops from old generation SATB buffers", _trashed_oops); } } @@ -124,19 +126,20 @@ class ShenandoahPurgeSATBTask : public WorkerTask { ShenandoahProcessOldSATB processor(mark_queue); while (satb_queues.apply_closure_to_completed_buffer(&processor)) {} - Atomic::add(&_trashed_oops, processor._trashed_oops); + Atomic::add(&_trashed_oops, processor.trashed_oops()); } }; class ShenandoahConcurrentCoalesceAndFillTask : public WorkerTask { - private: - uint _nworkers; - ShenandoahHeapRegion** _coalesce_and_fill_region_array; - uint _coalesce_and_fill_region_count; - volatile bool _is_preempted; - - public: - ShenandoahConcurrentCoalesceAndFillTask(uint nworkers, ShenandoahHeapRegion** coalesce_and_fill_region_array, +private: + uint _nworkers; + ShenandoahHeapRegion** _coalesce_and_fill_region_array; + uint _coalesce_and_fill_region_count; + volatile bool _is_preempted; + +public: + ShenandoahConcurrentCoalesceAndFillTask(uint nworkers, + ShenandoahHeapRegion** coalesce_and_fill_region_array, uint region_count) : WorkerTask("Shenandoah Concurrent Coalesce and Fill"), _nworkers(nworkers), @@ -149,7 +152,8 @@ class ShenandoahConcurrentCoalesceAndFillTask : public WorkerTask { for (uint region_idx = worker_id; region_idx < _coalesce_and_fill_region_count; region_idx += _nworkers) { ShenandoahHeapRegion* r = _coalesce_and_fill_region_array[region_idx]; if (r->is_humongous()) { - // there's only one object in this region and it's not garbage, so no need to coalesce or fill + // There is only one object in this region and it is not garbage, + // so no need to coalesce or fill. continue; } @@ -176,10 +180,6 @@ ShenandoahOldGeneration::ShenandoahOldGeneration(uint max_queues, size_t max_cap ref_processor()->set_soft_reference_policy(true); } -const char* ShenandoahOldGeneration::name() const { - return "OLD"; -} - bool ShenandoahOldGeneration::contains(ShenandoahHeapRegion* region) const { // TODO: Should this be region->is_old() instead? return !region->is_young(); @@ -205,7 +205,7 @@ bool ShenandoahOldGeneration::is_concurrent_mark_in_progress() { void ShenandoahOldGeneration::cancel_marking() { if (is_concurrent_mark_in_progress()) { - log_info(gc)("Abandon satb buffers."); + log_info(gc)("Abandon SATB buffers"); ShenandoahBarrierSet::satb_mark_queue_set().abandon_partial_marking(); } @@ -213,7 +213,6 @@ void ShenandoahOldGeneration::cancel_marking() { } void ShenandoahOldGeneration::prepare_gc() { - // Make the old generation regions parseable, so they can be safely // scanned when looking for objects in memory indicated by dirty cards. if (entry_coalesce_and_fill()) { @@ -231,17 +230,17 @@ void ShenandoahOldGeneration::prepare_gc() { } bool ShenandoahOldGeneration::entry_coalesce_and_fill() { - char msg[1024]; ShenandoahHeap* const heap = ShenandoahHeap::heap(); - ShenandoahConcurrentPhase gc_phase("Coalescing and filling (OLD)", ShenandoahPhaseTimings::coalesce_and_fill); + static const char* msg = "Coalescing and filling (OLD)"; + ShenandoahConcurrentPhase gc_phase(msg, ShenandoahPhaseTimings::coalesce_and_fill); // TODO: I don't think we're using these concurrent collection counters correctly. TraceCollectorStats tcs(heap->monitoring_support()->concurrent_collection_counters()); EventMark em("%s", msg); ShenandoahWorkerScope scope(heap->workers(), ShenandoahWorkerPolicy::calc_workers_for_conc_marking(), - "concurrent coalesce and fill"); + msg); return coalesce_and_fill(); } @@ -281,7 +280,7 @@ void ShenandoahOldGeneration::transfer_pointers_from_satb() { ShenandoahHeap* heap = ShenandoahHeap::heap(); shenandoah_assert_safepoint(); assert(heap->is_concurrent_old_mark_in_progress(), "Only necessary during old marking."); - log_info(gc)("Transfer satb buffers."); + log_info(gc)("Transfer SATB buffers"); uint nworkers = heap->workers()->active_workers(); StrongRootsScope scope(nworkers); @@ -298,7 +297,9 @@ void ShenandoahOldGeneration::prepare_regions_and_collection_set(bool concurrent assert(!heap->is_full_gc_in_progress(), "Only for concurrent and degenerated GC"); { - ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_update_region_states : ShenandoahPhaseTimings::degen_gc_final_update_region_states); + ShenandoahGCPhase phase(concurrent ? + ShenandoahPhaseTimings::final_update_region_states : + ShenandoahPhaseTimings::degen_gc_final_update_region_states); ShenandoahFinalMarkUpdateRegionStateClosure cl(complete_marking_context()); parallel_heap_region_iterate(&cl); @@ -308,7 +309,9 @@ void ShenandoahOldGeneration::prepare_regions_and_collection_set(bool concurrent { // This doesn't actually choose a collection set, but prepares a list of // regions as 'candidates' for inclusion in a mixed collection. - ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::choose_cset : ShenandoahPhaseTimings::degen_gc_choose_cset); + ShenandoahGCPhase phase(concurrent ? + ShenandoahPhaseTimings::choose_cset : + ShenandoahPhaseTimings::degen_gc_choose_cset); ShenandoahHeapLocker locker(heap->lock()); heuristics()->choose_collection_set(nullptr, nullptr); } @@ -316,7 +319,9 @@ void ShenandoahOldGeneration::prepare_regions_and_collection_set(bool concurrent { // Though we did not choose a collection set above, we still may have // freed up immediate garbage regions so proceed with rebuilding the free set. - ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_rebuild_freeset : ShenandoahPhaseTimings::degen_gc_final_rebuild_freeset); + ShenandoahGCPhase phase(concurrent ? + ShenandoahPhaseTimings::final_rebuild_freeset : + ShenandoahPhaseTimings::degen_gc_final_rebuild_freeset); ShenandoahHeapLocker locker(heap->lock()); heap->free_set()->rebuild(); } @@ -324,10 +329,10 @@ void ShenandoahOldGeneration::prepare_regions_and_collection_set(bool concurrent const char* ShenandoahOldGeneration::state_name(State state) { switch (state) { - case IDLE: return "Idle"; - case FILLING: return "Coalescing"; - case BOOTSTRAPPING: return "Bootstrapping"; - case MARKING: return "Marking"; + case IDLE: return "Idle"; + case FILLING: return "Coalescing"; + case BOOTSTRAPPING: return "Bootstrapping"; + case MARKING: return "Marking"; case WAITING_FOR_EVAC: return "Waiting for evacuation"; case WAITING_FOR_FILL: return "Waiting for fill"; default: @@ -339,7 +344,7 @@ const char* ShenandoahOldGeneration::state_name(State state) { void ShenandoahOldGeneration::transition_to(State new_state) { if (_state != new_state) { log_info(gc)("Old generation transition from %s to %s", state_name(_state), state_name(new_state)); - assert(validate_transition(new_state), "Invalid state transition."); + validate_transition(new_state); _state = new_state; } } @@ -395,7 +400,7 @@ void ShenandoahOldGeneration::transition_to(State new_state) { // +-----| FILLING | // +-----------------+ // -bool ShenandoahOldGeneration::validate_transition(State new_state) { +void ShenandoahOldGeneration::validate_transition(State new_state) { ShenandoahHeap* heap = ShenandoahHeap::heap(); switch (new_state) { case IDLE: @@ -404,32 +409,31 @@ bool ShenandoahOldGeneration::validate_transition(State new_state) { assert(_old_heuristics->unprocessed_old_collection_candidates() == 0, "Cannot become idle with collection candidates"); assert(!heap->is_prepare_for_old_mark_in_progress(), "Cannot become idle while making old generation parseable."); assert(heap->young_generation()->old_gen_task_queues() == nullptr, "Cannot become idle when setup for bootstrapping."); - return true; + break; case FILLING: assert(_state == IDLE || _state == WAITING_FOR_FILL, "Cannot begin filling without first completing evacuations, state is '%s'", state_name(_state)); assert(heap->is_prepare_for_old_mark_in_progress(), "Should be preparing for old mark now."); - return true; + break; case BOOTSTRAPPING: assert(_state == FILLING, "Cannot reset bitmap without making old regions parseable, state is '%s'", state_name(_state)); assert(_old_heuristics->unprocessed_old_collection_candidates() == 0, "Cannot bootstrap with mixed collection candidates"); assert(!heap->is_prepare_for_old_mark_in_progress(), "Cannot still be making old regions parseable."); - return true; + break; case MARKING: assert(_state == BOOTSTRAPPING, "Must have finished bootstrapping before marking, state is '%s'", state_name(_state)); assert(heap->young_generation()->old_gen_task_queues() != nullptr, "Young generation needs old mark queues."); assert(heap->is_concurrent_old_mark_in_progress(), "Should be marking old now."); - return true; + break; case WAITING_FOR_EVAC: assert(_state == MARKING, "Cannot have old collection candidates without first marking, state is '%s'", state_name(_state)); assert(_old_heuristics->unprocessed_old_collection_candidates() > 0, "Must have collection candidates here."); - return true; + break; case WAITING_FOR_FILL: assert(_state == MARKING || _state == WAITING_FOR_EVAC, "Cannot begin filling without first marking or evacuating, state is '%s'", state_name(_state)); assert(_old_heuristics->unprocessed_old_collection_candidates() > 0, "Cannot wait for fill without something to fill."); - return true; + break; default: - ShouldNotReachHere(); - return false; + fatal("Unknown new state"); } } #endif diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp index 6450f1fd39faf..0e3fb429c2634 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp @@ -32,28 +32,35 @@ class ShenandoahHeapRegionClosure; class ShenandoahOldHeuristics; class ShenandoahOldGeneration : public ShenandoahGeneration { - public: - ShenandoahOldGeneration(uint max_queues, size_t max_capacity, size_t soft_max_capacity); +private: + ShenandoahHeapRegion** _coalesce_and_fill_region_array; + ShenandoahOldHeuristics* _old_heuristics; - const char* name() const override; + bool entry_coalesce_and_fill(); + bool coalesce_and_fill(); - bool contains(ShenandoahHeapRegion* region) const override; +public: + ShenandoahOldGeneration(uint max_queues, size_t max_capacity, size_t soft_max_capacity); - bool contains(oop obj) const override; + virtual ShenandoahHeuristics* initialize_heuristics(ShenandoahMode* gc_mode) override; - void parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl) override; + const char* name() const override { + return "OLD"; + } + void parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl) override; void heap_region_iterate(ShenandoahHeapRegionClosure* cl) override; - void set_concurrent_mark_in_progress(bool in_progress) override; + bool contains(ShenandoahHeapRegion* region) const override; + bool contains(oop obj) const override; - virtual void cancel_marking() override; + void set_concurrent_mark_in_progress(bool in_progress) override; + bool is_concurrent_mark_in_progress() override; virtual void prepare_gc() override; - void prepare_regions_and_collection_set(bool concurrent) override; - - virtual ShenandoahHeuristics* initialize_heuristics(ShenandoahMode* gc_mode) override; + virtual void record_success_concurrent(bool abbreviated) override; + virtual void cancel_marking() override; // We leave the SATB barrier on for the entirety of the old generation // marking phase. In some cases, this can cause a write to a perfectly @@ -75,22 +82,15 @@ class ShenandoahOldGeneration : public ShenandoahGeneration { // the performance impact would be too severe. void transfer_pointers_from_satb(); - bool is_concurrent_mark_in_progress() override; - - virtual void record_success_concurrent(bool abbreviated) override; - +public: enum State { IDLE, FILLING, BOOTSTRAPPING, MARKING, WAITING_FOR_EVAC, WAITING_FOR_FILL }; - static const char* state_name(State state); - - void transition_to(State new_state); - -#ifdef ASSERT - bool validate_transition(State new_state); -#endif +private: + State _state; +public: State state() const { return _state; } @@ -99,13 +99,10 @@ class ShenandoahOldGeneration : public ShenandoahGeneration { return _state == IDLE || _state == WAITING_FOR_FILL; } - private: - bool entry_coalesce_and_fill(); - bool coalesce_and_fill(); + static const char* state_name(State state); - ShenandoahHeapRegion** _coalesce_and_fill_region_array; - ShenandoahOldHeuristics* _old_heuristics; - State _state; + void transition_to(State new_state); + void validate_transition(State new_state) NOT_DEBUG_RETURN; }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp index 4ab5863923f1f..f040cfe5e8ef8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.hpp @@ -52,7 +52,7 @@ class ShenandoahMarkRefsSuperClosure : public MetadataVisitingOopIterateClosure void work(T *p); public: - ShenandoahMarkRefsSuperClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp, ShenandoahObjToScanQueue* old_queue = nullptr); + ShenandoahMarkRefsSuperClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp, ShenandoahObjToScanQueue* old_q); bool is_weak() const { return _weak; @@ -76,8 +76,8 @@ class ShenandoahMarkUpdateRefsSuperClosure : public ShenandoahMarkRefsSuperClosu inline void work(T* p); public: - ShenandoahMarkUpdateRefsSuperClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp, ShenandoahObjToScanQueue* old = nullptr) : - ShenandoahMarkRefsSuperClosure(q, rp, old), + ShenandoahMarkUpdateRefsSuperClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp, ShenandoahObjToScanQueue* old_q) : + ShenandoahMarkRefsSuperClosure(q, rp, old_q), _heap(ShenandoahHeap::heap()) { assert(_heap->is_stw_gc_in_progress(), "Can only be used for STW GC"); }; @@ -90,8 +90,8 @@ class ShenandoahMarkUpdateRefsClosure : public ShenandoahMarkUpdateRefsSuperClos inline void do_oop_work(T* p) { work(p); } public: - ShenandoahMarkUpdateRefsClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp, ShenandoahObjToScanQueue* old = nullptr) : - ShenandoahMarkUpdateRefsSuperClosure(q, rp, old) {} + ShenandoahMarkUpdateRefsClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp, ShenandoahObjToScanQueue* old_q) : + ShenandoahMarkUpdateRefsSuperClosure(q, rp, old_q) {} virtual void do_oop(narrowOop* p) { do_oop_work(p); } virtual void do_oop(oop* p) { do_oop_work(p); } @@ -104,8 +104,8 @@ class ShenandoahMarkRefsClosure : public ShenandoahMarkRefsSuperClosure { inline void do_oop_work(T* p) { work(p); } public: - ShenandoahMarkRefsClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp, ShenandoahObjToScanQueue* old = nullptr) : - ShenandoahMarkRefsSuperClosure(q, rp, old) {}; + ShenandoahMarkRefsClosure(ShenandoahObjToScanQueue* q, ShenandoahReferenceProcessor* rp, ShenandoahObjToScanQueue* old_q) : + ShenandoahMarkRefsSuperClosure(q, rp, old_q) {}; virtual void do_oop(narrowOop* p) { do_oop_work(p); } virtual void do_oop(oop* p) { do_oop_work(p); } @@ -145,42 +145,21 @@ class ShenandoahConcUpdateRefsClosure : public ShenandoahUpdateRefsSuperClosure virtual void do_oop(oop* p) { work(p); } }; -class ShenandoahVerifyRemSetClosure : public BasicOopIterateClosure { - protected: - bool _init_mark; - ShenandoahHeap* _heap; - RememberedScanner* _scanner; - - public: -// Argument distinguishes between initial mark or start of update refs verification. - ShenandoahVerifyRemSetClosure(bool init_mark) : - _init_mark(init_mark), - _heap(ShenandoahHeap::heap()), - _scanner(_heap->card_scan()) { } - template - inline void work(T* p); - - virtual void do_oop(narrowOop* p) { work(p); } - virtual void do_oop(oop* p) { work(p); } -}; - class ShenandoahSetRememberedCardsToDirtyClosure : public BasicOopIterateClosure { - protected: - ShenandoahHeap* _heap; - RememberedScanner* _scanner; + ShenandoahHeap* const _heap; + RememberedScanner* const _scanner; public: - ShenandoahSetRememberedCardsToDirtyClosure() : _heap(ShenandoahHeap::heap()), - _scanner(_heap->card_scan()) { } + _scanner(_heap->card_scan()) {} template inline void work(T* p); virtual void do_oop(narrowOop* p) { work(p); } - virtual void do_oop(oop* p) { work(p); } + virtual void do_oop(oop* p) { work(p); } }; #endif // SHARE_GC_SHENANDOAH_SHENANDOAHOOPCLOSURES_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp index 6846dd8e09637..d257e91b4a23c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOopClosures.inline.hpp @@ -55,24 +55,6 @@ inline void ShenandoahConcUpdateRefsClosure::work(T* p) { _heap->conc_update_with_forwarded(p); } -template -inline void ShenandoahVerifyRemSetClosure::work(T* p) { - T o = RawAccess<>::oop_load(p); - if (!CompressedOops::is_null(o)) { - oop obj = CompressedOops::decode_not_null(o); - if (_heap->is_in_young(obj)) { - size_t card_index = _scanner->card_index_for_addr((HeapWord*) p); - if (_init_mark && !_scanner->is_card_dirty(card_index)) { - ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, nullptr, - "Verify init-mark remembered set violation", "clean card should be dirty", __FILE__, __LINE__); - } else if (!_init_mark && !_scanner->is_write_card_dirty(card_index)) { - ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, nullptr, - "Verify init-update-refs remembered set violation", "clean card should be dirty", __FILE__, __LINE__); - } - } - } -} - template inline void ShenandoahSetRememberedCardsToDirtyClosure::work(T* p) { T o = RawAccess<>::oop_load(p); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp index 48a7ec12d9c92..4bf9ed3e7726b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahPhaseTimings.hpp @@ -45,7 +45,7 @@ class outputStream; f(CNT_PREFIX ## CLDUnlink, DESC_PREFIX "Unlink CLDs") \ f(CNT_PREFIX ## WeakRefProc, DESC_PREFIX "Weak References") \ f(CNT_PREFIX ## ParallelMark, DESC_PREFIX "Parallel Mark") \ - f(CNT_PREFIX ## ScanClusters, DESC_PREFIX "Scan Clusters") + f(CNT_PREFIX ## ScanClusters, DESC_PREFIX "Scan Clusters") \ // end #define SHENANDOAH_PHASE_DO(f) \ @@ -261,8 +261,10 @@ class ShenandoahWorkerTimingsTracker : public StackObj { double _start_time; EventGCPhaseParallel _event; public: - ShenandoahWorkerTimingsTracker(ShenandoahPhaseTimings::Phase phase, ShenandoahPhaseTimings::ParPhase par_phase, - uint worker_id, bool cumulative = false); + ShenandoahWorkerTimingsTracker(ShenandoahPhaseTimings::Phase phase, + ShenandoahPhaseTimings::ParPhase par_phase, + uint worker_id, + bool cumulative = false); ~ShenandoahWorkerTimingsTracker(); }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp index f4498e415b0c2..a51a2643edaf9 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp @@ -40,24 +40,24 @@ template class ShenandoahInitMarkRootsClosure : public OopClosure { - private: +private: ShenandoahObjToScanQueue* const _queue; ShenandoahMarkingContext* const _mark_context; template - inline void do_oop_work(T* p); + inline void do_oop_work(T* p); - public: +public: ShenandoahInitMarkRootsClosure(ShenandoahObjToScanQueue* q); void do_oop(narrowOop* p) { do_oop_work(p); } void do_oop(oop* p) { do_oop_work(p); } }; -template +template ShenandoahInitMarkRootsClosure::ShenandoahInitMarkRootsClosure(ShenandoahObjToScanQueue* q) : -_queue(q), -_mark_context(ShenandoahHeap::heap()->marking_context()) { + _queue(q), + _mark_context(ShenandoahHeap::heap()->marking_context()) { } template diff --git a/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp b/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp index 191a9725ca0d8..68a10750e78c3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahThreadLocalData.hpp @@ -97,7 +97,6 @@ class ShenandoahThreadLocalData { // TODO: Preserve these stats somewhere for mutator threads. delete _evacuation_stats; - _evacuation_stats = nullptr; } static ShenandoahThreadLocalData* data(Thread* thread) { @@ -208,6 +207,7 @@ class ShenandoahThreadLocalData { } static void subtract_from_plab_evacuated(Thread* thread, size_t increment) { + // TODO: Assert underflow data(thread)->_plab_evacuated -= increment; } @@ -224,6 +224,7 @@ class ShenandoahThreadLocalData { } static void subtract_from_plab_promoted(Thread* thread, size_t increment) { + // TODO: Assert underflow data(thread)->_plab_promoted -= increment; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahUnload.cpp b/src/hotspot/share/gc/shenandoah/shenandoahUnload.cpp index b8f85436c840d..1b7e598b62f53 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahUnload.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahUnload.cpp @@ -50,6 +50,7 @@ class ShenandoahIsUnloadingOopClosure : public OopClosure { public: ShenandoahIsUnloadingOopClosure() : + // TODO: In non-generational mode, this should still be complete_marking_context() _marking_context(ShenandoahHeap::heap()->marking_context()), _is_unloading(false) { } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp index 8f8a0bf0b0db3..718d2f8766910 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp @@ -92,7 +92,8 @@ void VM_ShenandoahFinalRoots::doit() { HeapWord* top = r->top(); if (top > tams) { r->reset_age(); - } else if (heap->is_aging_cycle()){ + } else if (heap->is_aging_cycle()) { + // TODO: Does _incr_region_ages imply heap->is_aging_cycle()? r->increment_age(); } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp index fc6a50e66169e..716e4d9566e82 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp @@ -62,7 +62,7 @@ class VM_ShenandoahInitMark: public VM_ShenandoahOperation { private: ShenandoahConcurrentGC* const _gc; public: - explicit VM_ShenandoahInitMark(ShenandoahConcurrentGC* gc) : + VM_ShenandoahInitMark(ShenandoahConcurrentGC* gc) : VM_ShenandoahOperation(), _gc(gc) {}; VM_Operation::VMOp_Type type() const { return VMOp_ShenandoahInitMark; } @@ -74,7 +74,7 @@ class VM_ShenandoahFinalMarkStartEvac: public VM_ShenandoahOperation { private: ShenandoahConcurrentGC* const _gc; public: - explicit VM_ShenandoahFinalMarkStartEvac(ShenandoahConcurrentGC* gc) : + VM_ShenandoahFinalMarkStartEvac(ShenandoahConcurrentGC* gc) : VM_ShenandoahOperation(), _gc(gc) {}; VM_Operation::VMOp_Type type() const { return VMOp_ShenandoahFinalMarkStartEvac; } @@ -133,11 +133,12 @@ class VM_ShenandoahFinalUpdateRefs: public VM_ShenandoahOperation { class VM_ShenandoahFinalRoots: public VM_ShenandoahOperation { ShenandoahConcurrentGC* const _gc; - bool _incr_region_ages; + const bool _incr_region_ages; public: VM_ShenandoahFinalRoots(ShenandoahConcurrentGC* gc, bool incr_region_ages) : VM_ShenandoahOperation(), - _gc(gc), _incr_region_ages(incr_region_ages) {}; + _gc(gc), + _incr_region_ages(incr_region_ages) {}; VM_Operation::VMOp_Type type() const { return VMOp_ShenandoahFinalRoots; } const char* name() const { return "Shenandoah Final Roots"; } virtual void doit(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp index 5a63c4e89e7e7..b6a1203683bec 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp @@ -118,7 +118,9 @@ class ShenandoahVerifyOopClosure : public BasicOopIterateClosure { // For performance reasons, only fully verify non-marked field values. // We are here when the host object for *p is already marked. - if ( in_generation(obj) && _map->par_mark(obj)) { + // TODO: We should consider specializing this closure by generation ==/!= null, + // to avoid in_generation check on fast path here for non-generational mode. + if (in_generation(obj) && _map->par_mark(obj)) { verify_oop_at(p, obj); _stack->push(ShenandoahVerifierTask(obj)); } @@ -144,7 +146,7 @@ class ShenandoahVerifyOopClosure : public BasicOopIterateClosure { check(ShenandoahAsserts::_safe_unknown, obj, is_object_aligned(obj), "oop must be aligned"); - ShenandoahHeapRegion* obj_reg = _heap->heap_region_containing(obj); + ShenandoahHeapRegion *obj_reg = _heap->heap_region_containing(obj); Klass* obj_klass = obj->klass_or_null(); // Verify that obj is not in dead space: @@ -155,7 +157,7 @@ class ShenandoahVerifyOopClosure : public BasicOopIterateClosure { check(ShenandoahAsserts::_safe_unknown, obj, Metaspace::contains(obj_klass), "Object klass pointer must go to metaspace"); - HeapWord* obj_addr = cast_from_oop(obj); + HeapWord *obj_addr = cast_from_oop(obj); check(ShenandoahAsserts::_safe_unknown, obj, obj_addr < obj_reg->top(), "Object start should be within the region"); @@ -220,7 +222,7 @@ class ShenandoahVerifyOopClosure : public BasicOopIterateClosure { check(ShenandoahAsserts::_safe_oop, obj, !fwd_reg->is_humongous(), "Should have no humongous forwardees"); - HeapWord* fwd_addr = cast_from_oop(fwd); + HeapWord *fwd_addr = cast_from_oop(fwd); check(ShenandoahAsserts::_safe_oop, obj, fwd_addr < fwd_reg->top(), "Forwardee start should be within the region"); check(ShenandoahAsserts::_safe_oop, obj, (fwd_addr + fwd->size()) <= fwd_reg->top(), @@ -351,11 +353,11 @@ class ShenandoahCalculateRegionStatsClosure : public ShenandoahHeapRegionClosure void heap_region_do(ShenandoahHeapRegion* r) { _used += r->used(); - log_debug(gc)("ShenandoahCalculateRegionStatsClosure added " SIZE_FORMAT " for %s Region " SIZE_FORMAT ", yielding: " SIZE_FORMAT, - r->used(), r->is_humongous()? "humongous": "regular", r->index(), _used); _garbage += r->garbage(); _committed += r->is_committed() ? ShenandoahHeapRegion::region_size_bytes() : 0; _regions++; + log_debug(gc)("ShenandoahCalculateRegionStatsClosure: adding " SIZE_FORMAT " for %s Region " SIZE_FORMAT ", yielding: " SIZE_FORMAT, + r->used(), (r->is_humongous() ? "humongous" : "regular"), r->index(), _used); } size_t used() { return _used; } @@ -375,25 +377,26 @@ class ShenandoahGenerationStatsClosure : public ShenandoahHeapRegionClosure { void heap_region_do(ShenandoahHeapRegion* r) override { switch (r->affiliation()) { - default: - ShouldNotReachHere(); + case FREE: return; - case FREE: return; case YOUNG_GENERATION: young.heap_region_do(r); + global.heap_region_do(r); break; case OLD_GENERATION: old.heap_region_do(r); + global.heap_region_do(r); break; + default: + ShouldNotReachHere(); } - global.heap_region_do(r); } static void log_usage(ShenandoahGeneration* generation, ShenandoahCalculateRegionStatsClosure& stats) { log_debug(gc)("Safepoint verification: %s verified usage: " SIZE_FORMAT "%s, recorded usage: " SIZE_FORMAT "%s", generation->name(), byte_size_in_proper_unit(generation->used()), proper_unit_for_byte_size(generation->used()), - byte_size_in_proper_unit(stats.used()), proper_unit_for_byte_size(stats.used())); + byte_size_in_proper_unit(stats.used()), proper_unit_for_byte_size(stats.used())); } static void validate_usage(const char* label, ShenandoahGeneration* generation, ShenandoahCalculateRegionStatsClosure& stats) { @@ -402,7 +405,7 @@ class ShenandoahGenerationStatsClosure : public ShenandoahHeapRegionClosure { "%s: generation (%s) used size must be consistent: generation-used = " SIZE_FORMAT "%s, regions-used = " SIZE_FORMAT "%s", label, generation->name(), byte_size_in_proper_unit(generation_used), proper_unit_for_byte_size(generation_used), - byte_size_in_proper_unit(stats.used()), proper_unit_for_byte_size(stats.used())); + byte_size_in_proper_unit(stats.used()), proper_unit_for_byte_size(stats.used())); guarantee(stats.regions() == generation->used_regions(), "%s: generation (%s) used regions (" SIZE_FORMAT ") must equal regions that are in use (" SIZE_FORMAT ")", @@ -579,7 +582,7 @@ class ShenandoahVerifierMarkedRegionTask : public WorkerTask { private: const char* _label; ShenandoahVerifier::VerifyOptions _options; - ShenandoahHeap* _heap; + ShenandoahHeap *_heap; MarkBitMap* _bitmap; ShenandoahLivenessData* _ld; volatile size_t _claimed; @@ -639,7 +642,7 @@ class ShenandoahVerifierMarkedRegionTask : public WorkerTask { return _generation == nullptr || _generation->contains(r); } - virtual void work_humongous(ShenandoahHeapRegion* r, ShenandoahVerifierStack& stack, ShenandoahVerifyOopClosure& cl) { + virtual void work_humongous(ShenandoahHeapRegion *r, ShenandoahVerifierStack& stack, ShenandoahVerifyOopClosure& cl) { size_t processed = 0; HeapWord* obj = r->bottom(); if (_heap->complete_marking_context()->is_marked(cast_to_oop(obj))) { @@ -648,7 +651,7 @@ class ShenandoahVerifierMarkedRegionTask : public WorkerTask { Atomic::add(&_processed, processed, memory_order_relaxed); } - virtual void work_regular(ShenandoahHeapRegion* r, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl) { + virtual void work_regular(ShenandoahHeapRegion *r, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl) { size_t processed = 0; ShenandoahMarkingContext* ctx = _heap->complete_marking_context(); HeapWord* tams = ctx->top_at_mark_start(r); @@ -681,7 +684,7 @@ class ShenandoahVerifierMarkedRegionTask : public WorkerTask { Atomic::add(&_processed, processed, memory_order_relaxed); } - void verify_and_follow(HeapWord* addr, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl, size_t* processed) { + void verify_and_follow(HeapWord *addr, ShenandoahVerifierStack &stack, ShenandoahVerifyOopClosure &cl, size_t *processed) { if (!_bitmap->par_mark(addr)) return; // Verify the object itself: @@ -711,10 +714,17 @@ class VerifyThreadGCState : public ThreadClosure { VerifyThreadGCState(const char* label, char expected) : _label(label), _expected(expected) {} void do_thread(Thread* t) { char actual = ShenandoahThreadLocalData::gc_state(t); - if (actual != _expected && !(actual & ShenandoahHeap::OLD_MARKING)) { + if (!verify_gc_state(actual, _expected)) { fatal("%s: Thread %s: expected gc-state %d, actual %d", _label, t->name(), _expected, actual); } } + + static bool verify_gc_state(char actual, char expected) { + // Old generation marking is allowed in all states. + // TODO: This actually accepts more than just OLD_MARKING. + // TODO: Also, only accept OLD_MARKING in generational mode. + return (actual == expected) || (actual & ShenandoahHeap::OLD_MARKING); + } }; void ShenandoahVerifier::verify_at_safepoint(const char* label, @@ -774,8 +784,7 @@ void ShenandoahVerifier::verify_at_safepoint(const char* label, if (enabled) { char actual = _heap->gc_state(); - // Old generation marking is allowed in all states. - if (actual != expected && !(actual & ShenandoahHeap::OLD_MARKING)) { + if (!VerifyThreadGCState::verify_gc_state(actual, expected)) { fatal("%s: Global gc-state: expected %d, actual %d", label, expected, actual); } @@ -821,33 +830,36 @@ void ShenandoahVerifier::verify_at_safepoint(const char* label, if (generation != nullptr) { ShenandoahHeapLocker lock(_heap->lock()); - if (remembered == _verify_remembered_for_marking) { - log_debug(gc)("Safepoint verification of remembered set at mark"); - } else if (remembered == _verify_remembered_for_updating_references) { - log_debug(gc)("Safepoint verification of remembered set at update ref"); - } else if (remembered == _verify_remembered_after_full_gc) { - log_debug(gc)("Safepoint verification of remembered set after full gc"); - } - - if (remembered == _verify_remembered_for_marking) { - _heap->verify_rem_set_at_mark(); - } else if (remembered == _verify_remembered_for_updating_references) { - _heap->verify_rem_set_at_update_ref(); - } else if (remembered == _verify_remembered_after_full_gc) { - _heap->verify_rem_set_after_full_gc(); + switch (remembered) { + case _verify_remembered_disable: + break; + case _verify_remembered_before_marking: + log_debug(gc)("Safepoint verification of remembered set at mark"); + verify_rem_set_before_mark(); + break; + case _verify_remembered_before_updating_references: + log_debug(gc)("Safepoint verification of remembered set at update ref"); + verify_rem_set_before_update_ref(); + break; + case _verify_remembered_after_full_gc: + log_debug(gc)("Safepoint verification of remembered set after full gc"); + verify_rem_set_after_full_gc(); + break; + default: + fatal("Unhandled remembered set verification mode"); } ShenandoahGenerationStatsClosure cl; _heap->heap_region_iterate(&cl); if (LogTarget(Debug, gc)::is_enabled()) { - ShenandoahGenerationStatsClosure::log_usage(_heap->old_generation(), cl.old); - ShenandoahGenerationStatsClosure::log_usage(_heap->young_generation(), cl.young); + ShenandoahGenerationStatsClosure::log_usage(_heap->old_generation(), cl.old); + ShenandoahGenerationStatsClosure::log_usage(_heap->young_generation(), cl.young); ShenandoahGenerationStatsClosure::log_usage(_heap->global_generation(), cl.global); } - ShenandoahGenerationStatsClosure::validate_usage(label, _heap->old_generation(), cl.old); - ShenandoahGenerationStatsClosure::validate_usage(label, _heap->young_generation(), cl.young); + ShenandoahGenerationStatsClosure::validate_usage(label, _heap->old_generation(), cl.old); + ShenandoahGenerationStatsClosure::validate_usage(label, _heap->young_generation(), cl.young); ShenandoahGenerationStatsClosure::validate_usage(label, _heap->global_generation(), cl.global); } @@ -966,8 +978,8 @@ void ShenandoahVerifier::verify_generic(VerifyOption vo) { void ShenandoahVerifier::verify_before_concmark() { verify_at_safepoint( - "Before Mark", - _verify_remembered_for_marking, // verify read-only remembered set from bottom() to top() + "Before Mark", + _verify_remembered_before_marking, // verify read-only remembered set from bottom() to top() _verify_forwarded_none, // UR should have fixed up _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations _verify_cset_none, // UR should have fixed this @@ -1032,7 +1044,7 @@ void ShenandoahVerifier::verify_after_evacuation() { void ShenandoahVerifier::verify_before_updaterefs() { verify_at_safepoint( "Before Updating References", - _verify_remembered_for_updating_references, // verify read-write remembered set + _verify_remembered_before_updating_references, // verify read-write remembered set _verify_forwarded_allow, // forwarded references allowed _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well _verify_cset_forwarded, // all cset refs are fully forwarded @@ -1081,23 +1093,10 @@ void ShenandoahVerifier::verify_before_fullgc() { ); } -void ShenandoahVerifier::verify_after_generational_fullgc() { - verify_at_safepoint( - "After Full Generational GC", - _verify_remembered_after_full_gc, // verify read-write remembered set - _verify_forwarded_none, // all objects are non-forwarded - _verify_marked_complete, // all objects are marked in complete bitmap - _verify_cset_none, // no cset references - _verify_liveness_disable, // no reliable liveness data anymore - _verify_regions_notrash_nocset, // no trash, no cset - _verify_gcstate_stable // full gc cleaned up everything - ); -} - void ShenandoahVerifier::verify_after_fullgc() { verify_at_safepoint( "After Full GC", - _verify_remembered_disable, // do not verify remembered set + _verify_remembered_after_full_gc, // verify read-write remembered set _verify_forwarded_none, // all objects are non-forwarded _verify_marked_complete, // all objects are marked in complete bitmap _verify_cset_none, // no cset references @@ -1107,6 +1106,7 @@ void ShenandoahVerifier::verify_after_fullgc() { ); } +// TODO: Why this closure does not visit metadata? class ShenandoahVerifyNoForwared : public BasicOopIterateClosure { private: template @@ -1127,6 +1127,7 @@ class ShenandoahVerifyNoForwared : public BasicOopIterateClosure { void do_oop(oop* p) { do_oop_work(p); } }; +// TODO: Why this closure does not visit metadata? class ShenandoahVerifyInToSpaceClosure : public BasicOopIterateClosure { private: template @@ -1168,3 +1169,215 @@ void ShenandoahVerifier::verify_roots_no_forwarded() { ShenandoahVerifyNoForwared cl; ShenandoahRootVerifier::roots_do(&cl); } + +class ShenandoahVerifyRemSetClosure : public BasicOopIterateClosure { +protected: + bool const _init_mark; + ShenandoahHeap* const _heap; + RememberedScanner* const _scanner; + +public: + // Argument distinguishes between initial mark or start of update refs verification. + ShenandoahVerifyRemSetClosure(bool init_mark) : + _init_mark(init_mark), + _heap(ShenandoahHeap::heap()), + _scanner(_heap->card_scan()) {} + + template + inline void work(T* p) { + T o = RawAccess<>::oop_load(p); + if (!CompressedOops::is_null(o)) { + oop obj = CompressedOops::decode_not_null(o); + if (_heap->is_in_young(obj)) { + size_t card_index = _scanner->card_index_for_addr((HeapWord*) p); + if (_init_mark && !_scanner->is_card_dirty(card_index)) { + ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, nullptr, + "Verify init-mark remembered set violation", "clean card should be dirty", __FILE__, __LINE__); + } else if (!_init_mark && !_scanner->is_write_card_dirty(card_index)) { + ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, p, nullptr, + "Verify init-update-refs remembered set violation", "clean card should be dirty", __FILE__, __LINE__); + } + } + } + } + + virtual void do_oop(narrowOop* p) { work(p); } + virtual void do_oop(oop* p) { work(p); } +}; + +void ShenandoahVerifier::help_verify_region_rem_set(ShenandoahHeapRegion* r, ShenandoahMarkingContext* ctx, HeapWord* from, + HeapWord* top, HeapWord* registration_watermark, const char* message) { + RememberedScanner* scanner = _heap->card_scan(); + ShenandoahVerifyRemSetClosure check_interesting_pointers(false); + + HeapWord* obj_addr = from; + if (r->is_humongous_start()) { + oop obj = cast_to_oop(obj_addr); + if ((ctx == nullptr) || ctx->is_marked(obj)) { + size_t card_index = scanner->card_index_for_addr(obj_addr); + // For humongous objects, the typical object is an array, so the following checks may be overkill + // For regular objects (not object arrays), if the card holding the start of the object is dirty, + // we do not need to verify that cards spanning interesting pointers within this object are dirty. + if (!scanner->is_write_card_dirty(card_index) || obj->is_objArray()) { + obj->oop_iterate(&check_interesting_pointers); + } + // else, object's start is marked dirty and obj is not an objArray, so any interesting pointers are covered + } + // else, this humongous object is not live so no need to verify its internal pointers + + if ((obj_addr < registration_watermark) && !scanner->verify_registration(obj_addr, ctx)) { + ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, obj_addr, nullptr, message, + "object not properly registered", __FILE__, __LINE__); + } + } else if (!r->is_humongous()) { + while (obj_addr < top) { + oop obj = cast_to_oop(obj_addr); + // ctx->is_marked() returns true if mark bit set or if obj above TAMS. + if ((ctx == nullptr) || ctx->is_marked(obj)) { + size_t card_index = scanner->card_index_for_addr(obj_addr); + // For regular objects (not object arrays), if the card holding the start of the object is dirty, + // we do not need to verify that cards spanning interesting pointers within this object are dirty. + if (!scanner->is_write_card_dirty(card_index) || obj->is_objArray()) { + obj->oop_iterate(&check_interesting_pointers); + } + // else, object's start is marked dirty and obj is not an objArray, so any interesting pointers are covered + + if ((obj_addr < registration_watermark) && !scanner->verify_registration(obj_addr, ctx)) { + ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, obj_addr, nullptr, message, + "object not properly registered", __FILE__, __LINE__); + } + obj_addr += obj->size(); + } else { + // This object is not live so we don't verify dirty cards contained therein + HeapWord* tams = ctx->top_at_mark_start(r); + obj_addr = ctx->get_next_marked_addr(obj_addr, tams); + } + } + } +} + +// Assure that the remember set has a dirty card everywhere there is an interesting pointer. +// This examines the read_card_table between bottom() and top() since all PLABS are retired +// before the safepoint for init_mark. Actually, we retire them before update-references and don't +// restore them until the start of evacuation. +void ShenandoahVerifier::verify_rem_set_before_mark() { + shenandoah_assert_safepoint(); + assert(_heap->mode()->is_generational(), "Only verify remembered set for generational operational modes"); + + ShenandoahRegionIterator iterator; + RememberedScanner* scanner = _heap->card_scan(); + ShenandoahVerifyRemSetClosure check_interesting_pointers(true); + ShenandoahMarkingContext* ctx; + + log_debug(gc)("Verifying remembered set at %s mark", _heap->doing_mixed_evacuations()? "mixed": "young"); + + if (_heap->is_old_bitmap_stable() || _heap->active_generation()->is_global()) { + ctx = _heap->complete_marking_context(); + } else { + ctx = nullptr; + } + + while (iterator.has_next()) { + ShenandoahHeapRegion* r = iterator.next(); + if (r == nullptr) { + // TODO: Can this really happen? + break; + } + + HeapWord* tams = (ctx != nullptr) ? ctx->top_at_mark_start(r) : nullptr; + + // TODO: Is this replaceable with call to help_verify_region_rem_set? + + if (r->is_old() && r->is_active()) { + HeapWord* obj_addr = r->bottom(); + if (r->is_humongous_start()) { + oop obj = cast_to_oop(obj_addr); + if ((ctx == nullptr) || ctx->is_marked(obj)) { + // For humongous objects, the typical object is an array, so the following checks may be overkill + // For regular objects (not object arrays), if the card holding the start of the object is dirty, + // we do not need to verify that cards spanning interesting pointers within this object are dirty. + if (!scanner->is_card_dirty(obj_addr) || obj->is_objArray()) { + obj->oop_iterate(&check_interesting_pointers); + } + // else, object's start is marked dirty and obj is not an objArray, so any interesting pointers are covered + } + // else, this humongous object is not marked so no need to verify its internal pointers + if (!scanner->verify_registration(obj_addr, ctx)) { + ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, obj_addr, nullptr, + "Verify init-mark remembered set violation", "object not properly registered", __FILE__, __LINE__); + } + } else if (!r->is_humongous()) { + HeapWord* top = r->top(); + while (obj_addr < top) { + oop obj = cast_to_oop(obj_addr); + // ctx->is_marked() returns true if mark bit set (TAMS not relevant during init mark) + if ((ctx == nullptr) || ctx->is_marked(obj)) { + // For regular objects (not object arrays), if the card holding the start of the object is dirty, + // we do not need to verify that cards spanning interesting pointers within this object are dirty. + if (!scanner->is_card_dirty(obj_addr) || obj->is_objArray()) { + obj->oop_iterate(&check_interesting_pointers); + } + // else, object's start is marked dirty and obj is not an objArray, so any interesting pointers are covered + if (!scanner->verify_registration(obj_addr, ctx)) { + ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, obj_addr, nullptr, + "Verify init-mark remembered set violation", "object not properly registered", __FILE__, __LINE__); + } + obj_addr += obj->size(); + } else { + // This object is not live so we don't verify dirty cards contained therein + assert(tams != nullptr, "If object is not live, ctx and tams should be non-null"); + obj_addr = ctx->get_next_marked_addr(obj_addr, tams); + } + } + } // else, we ignore humongous continuation region + } // else, this is not an OLD region so we ignore it + } // all regions have been processed +} + +void ShenandoahVerifier::verify_rem_set_after_full_gc() { + shenandoah_assert_safepoint(); + assert(_heap->mode()->is_generational(), "Only verify remembered set for generational operational modes"); + + ShenandoahRegionIterator iterator; + + while (iterator.has_next()) { + ShenandoahHeapRegion* r = iterator.next(); + if (r == nullptr) { + // TODO: Can this really happen? + break; + } + if (r->is_old() && !r->is_cset()) { + help_verify_region_rem_set(r, nullptr, r->bottom(), r->top(), r->top(), "Remembered set violation at end of Full GC"); + } + } +} + +// Assure that the remember set has a dirty card everywhere there is an interesting pointer. Even though +// the update-references scan of remembered set only examines cards up to update_watermark, the remembered +// set should be valid through top. This examines the write_card_table between bottom() and top() because +// all PLABS are retired immediately before the start of update refs. +void ShenandoahVerifier::verify_rem_set_before_update_ref() { + shenandoah_assert_safepoint(); + assert(_heap->mode()->is_generational(), "Only verify remembered set for generational operational modes"); + + ShenandoahRegionIterator iterator; + ShenandoahMarkingContext* ctx; + + if (_heap->is_old_bitmap_stable() || _heap->active_generation()->is_global()) { + ctx = _heap->complete_marking_context(); + } else { + ctx = nullptr; + } + + while (iterator.has_next()) { + ShenandoahHeapRegion* r = iterator.next(); + if (r == nullptr) { + // TODO: Can this really happen? + break; + } + if (r->is_old() && !r->is_cset()) { + help_verify_region_rem_set(r, ctx, r->bottom(), r->top(), r->get_update_watermark(), + "Remembered set violation at init-update-references"); + } + } +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp index 7dfe55e46430d..284746be54108 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp @@ -62,17 +62,17 @@ class ShenandoahVerifier : public CHeapObj { // Disable remembered set verification. _verify_remembered_disable, - // Assure old objects are registered and remembered set cards within the read-only remembered set are dirty - // for every interesting pointer within each OLD ShenandoahHeapRegion between bottom() and top(). This is - // appropriate at the init_mark safepoint since all TLABS are retired before we reach this code. - _verify_remembered_for_marking, + // Old objects should be registered and RS cards within *read-only* RS are dirty for all + // inter-generational pointers. + _verify_remembered_before_marking, - // Assure old objects are registered and remembered set cards within the read-write remembered set are dirty - // for every interesting pointer within each OLD ShenandoahHeapRegion between bottom() and top(). - _verify_remembered_for_updating_references, + // Old objects should be registered and RS cards within *read-write* RS are dirty for all + // inter-generational pointers. + _verify_remembered_before_updating_references, - // Assure old objects are registered and remembered set cards within the read-write remembered set are dirty - // for every interesting pointer within each OLD ShenandoahHeapRegion between bottom() and top(). + // Old objects should be registered and RS cards within *read-write* RS are dirty for all + // inter-generational pointers. + // TODO: This differs from the previous mode by update-watermark() vs top() end range? _verify_remembered_after_full_gc } VerifyRememberedSet; @@ -157,7 +157,7 @@ class ShenandoahVerifier : public CHeapObj { // Evacuation is in progress, some objects are forwarded _verify_gcstate_evacuation, - // Evacuation is done, objects are forwarded, updating is in progress + // Evacuation is done, some objects are forwarded, updating is in progress _verify_gcstate_updating } VerifyGCState; @@ -204,7 +204,6 @@ class ShenandoahVerifier : public CHeapObj { void verify_after_updaterefs(); void verify_before_fullgc(); void verify_after_fullgc(); - void verify_after_generational_fullgc(); void verify_after_degenerated(); void verify_generic(VerifyOption option); @@ -212,6 +211,14 @@ class ShenandoahVerifier : public CHeapObj { void verify_roots_in_to_space(); void verify_roots_no_forwarded(); + +private: + void help_verify_region_rem_set(ShenandoahHeapRegion* r, ShenandoahMarkingContext* ctx, + HeapWord* from, HeapWord* top, HeapWord* update_watermark, const char* message); + + void verify_rem_set_before_mark(); + void verify_rem_set_before_update_ref(); + void verify_rem_set_after_full_gc(); }; #endif // SHARE_GC_SHENANDOAH_SHENANDOAHVERIFIER_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp index 632a3df67d52e..bb2e1f2e493b5 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.cpp @@ -37,10 +37,6 @@ ShenandoahYoungGeneration::ShenandoahYoungGeneration(uint max_queues, size_t max _old_gen_task_queues(nullptr) { } -const char* ShenandoahYoungGeneration::name() const { - return "YOUNG"; -} - void ShenandoahYoungGeneration::set_concurrent_mark_in_progress(bool in_progress) { ShenandoahHeap* heap = ShenandoahHeap::heap(); heap->set_concurrent_young_mark_in_progress(in_progress); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp index 4621ed6eea1fb..897cc920cd344 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahYoungGeneration.hpp @@ -34,20 +34,25 @@ class ShenandoahYoungGeneration : public ShenandoahGeneration { public: ShenandoahYoungGeneration(uint max_queues, size_t max_capacity, size_t max_soft_capacity); - const char* name() const override; + virtual ShenandoahHeuristics* initialize_heuristics(ShenandoahMode* gc_mode) override; + + const char* name() const override { + return "YOUNG"; + } void set_concurrent_mark_in_progress(bool in_progress) override; - void parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl) override; + bool is_concurrent_mark_in_progress() override; + void parallel_heap_region_iterate(ShenandoahHeapRegionClosure* cl) override; void heap_region_iterate(ShenandoahHeapRegionClosure* cl) override; bool contains(ShenandoahHeapRegion* region) const override; bool contains(oop obj) const override; + void reserve_task_queues(uint workers) override; void set_old_gen_task_queues(ShenandoahObjToScanQueueSet* old_gen_queues) { _old_gen_task_queues = old_gen_queues; } - ShenandoahObjToScanQueueSet* old_gen_task_queues() const override { return _old_gen_task_queues; } @@ -58,15 +63,7 @@ class ShenandoahYoungGeneration : public ShenandoahGeneration { return _old_gen_task_queues != nullptr; } - void reserve_task_queues(uint workers) override; - - virtual ShenandoahHeuristics* initialize_heuristics(ShenandoahMode* gc_mode) override; - virtual void add_collection_time(double time_seconds) override; - - protected: - bool is_concurrent_mark_in_progress() override; - }; #endif // SHARE_VM_GC_SHENANDOAH_SHENANDOAHYOUNGGENERATION_HPP From b26235005087cb43fc0871efc777e09881a8f07e Mon Sep 17 00:00:00 2001 From: Aleksey Shipilev Date: Thu, 13 Apr 2023 08:39:32 +0000 Subject: [PATCH 214/254] Reduce the number of new product flags Reviewed-by: ysr, rkennke --- .../share/gc/shenandoah/shenandoah_globals.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp index 0439fad44b56f..ab2e97029c06a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp @@ -78,7 +78,7 @@ " compact - run GC more frequently and with deeper targets to " \ "free up more memory.") \ \ - product(ccstr, ShenandoahOldGCHeuristics, "adaptive", \ + product(ccstr, ShenandoahOldGCHeuristics, "adaptive", EXPERIMENTAL, \ "Similar to ShenandoahGCHeuristics, but applied to the old " \ "generation. This configuration is only used to trigger old " \ "collections and does not change how regions are selected " \ @@ -205,13 +205,13 @@ "time from active application. Time is in milliseconds. " \ "Setting this to 0 disables the feature.") \ \ - product(uintx, ShenandoahGuaranteedOldGCInterval, 10*60*1000, EXPERIMENTAL, \ + product(uintx, ShenandoahGuaranteedOldGCInterval, 10*60*1000, EXPERIMENTAL, \ "Run a collection of the old generation at least this often. " \ "Heuristics may trigger collections more frequently. Time is in " \ "milliseconds. Setting this to 0 disables the feature.") \ \ - product(uintx, ShenandoahGuaranteedYoungGCInterval, 5*60*1000, EXPERIMENTAL, \ - "Run a collection of the young generation at least this often. " \ + product(uintx, ShenandoahGuaranteedYoungGCInterval, 5*60*1000, EXPERIMENTAL, \ + "Run a collection of the young generation at least this often. " \ "Heuristics may trigger collections more frequently. Time is in " \ "milliseconds. Setting this to 0 disables the feature.") \ \ @@ -371,13 +371,13 @@ "events.") \ range(0,100) \ \ - product(uintx, ShenandoahMinYoungPercentage, 20, \ + product(uintx, ShenandoahMinYoungPercentage, 20, EXPERIMENTAL, \ "The minimum percentage of the heap to use for the young " \ "generation. Heuristics will not adjust the young generation " \ "to be less than this.") \ range(0, 100) \ \ - product(uintx, ShenandoahMaxYoungPercentage, 80, \ + product(uintx, ShenandoahMaxYoungPercentage, 80, EXPERIMENTAL, \ "The maximum percentage of the heap to use for the young " \ "generation. Heuristics will not adjust the young generation " \ "to be more than this.") \ From 7df41365a86782bbced39efcf21403e7b967df06 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Fri, 14 Apr 2023 16:44:09 +0000 Subject: [PATCH 215/254] Disable assertions in code that is expected to be replaced soon Reviewed-by: kdnilsen --- src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp | 6 ++++-- src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index 427dc41c969dd..9b1569bf858c4 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -261,7 +261,8 @@ void ShenandoahGeneration::compute_evacuation_budgets(ShenandoahHeap* heap, bool // 3. ((young_gen->capacity * ShenandoahEvacReserve / 100) * ShenandoahOldEvacRatioPercent) / 100 // (e.g. old evacuation should be no larger than 12% of young-gen evacuation) old_evacuation_reserve = old_generation->available(); - assert(old_evacuation_reserve > minimum_evacuation_reserve, "Old-gen available has not been preserved!"); + // This assertion has been disabled because we expect this code to be replaced by 05/2023 + // assert(old_evacuation_reserve > minimum_evacuation_reserve, "Old-gen available has not been preserved!"); size_t old_evac_reserve_max = old_generation->soft_max_capacity() * ShenandoahOldEvacReserve / 100; if (old_evac_reserve_max < old_evacuation_reserve) { old_evacuation_reserve = old_evac_reserve_max; @@ -1027,7 +1028,8 @@ size_t ShenandoahGeneration::adjusted_capacity() const { } size_t ShenandoahGeneration::adjusted_unaffiliated_regions() const { - assert(adjusted_capacity() >= used_regions_size(), "adjusted_unaffiliated_regions() cannot return negative"); + // This assertion has been disabled because we expect this code to be replaced by 05/2023 + // assert(adjusted_capacity() >= used_regions_size(), "adjusted_unaffiliated_regions() cannot return negative"); assert((adjusted_capacity() - used_regions_size()) % ShenandoahHeapRegion::region_size_bytes() == 0, "adjusted capacity (" SIZE_FORMAT ") and used regions size (" SIZE_FORMAT ") should be multiples of region_size_bytes", adjusted_capacity(), used_regions_size()); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 57a14962ed068..853e697bbf48e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -1304,7 +1304,8 @@ HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req // When we retire this plab, we'll unexpend what we don't really use. ShenandoahThreadLocalData::enable_plab_promotions(thread); expend_promoted(actual_size); - assert(get_promoted_expended() <= get_promoted_reserve(), "Do not expend more promotion than budgeted"); + // This assert has been disabled because we expect this code to be replaced by 05/2023. + // assert(get_promoted_expended() <= get_promoted_reserve(), "Do not expend more promotion than budgeted"); ShenandoahThreadLocalData::set_plab_preallocated_promoted(thread, actual_size); } else { // Disable promotions in this thread because entirety of this PLAB must be available to hold old-gen evacuations. From b5e2102a715e9392d5a10d8609c91049c360e1a4 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Mon, 17 Apr 2023 22:22:01 +0000 Subject: [PATCH 216/254] Disable failing tests while fixes are in progress Reviewed-by: ysr, shade --- test/hotspot/jtreg/ProblemList.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/hotspot/jtreg/ProblemList.txt b/test/hotspot/jtreg/ProblemList.txt index 3aad87ff75d4f..62a173f7daffb 100644 --- a/test/hotspot/jtreg/ProblemList.txt +++ b/test/hotspot/jtreg/ProblemList.txt @@ -83,6 +83,17 @@ gc/stress/gclocker/TestGCLockerWithG1.java 8180622 generic-all gc/stress/TestJNIBlockFullGC/TestJNIBlockFullGC.java 8192647 generic-all gc/stress/TestStressG1Humongous.java 8286554 windows-x64 +gc/shenandoah/TestDynamicSoftMaxHeapSize.java#generational 8306024 generic-all +gc/shenandoah/TestArrayCopyStress.java#generational 8306024 generic-all +gc/shenandoah/oom/TestThreadFailure.java 8306024 generic-all +gc/shenandoah/oom/TestClassLoaderLeak 8306024 generic-all +gc/shenandoah/mxbeans/TestChurnNotifications#generational 8306024 generic-all +gc/stress/gcold/TestGCOldWithShenandoah.java#generational 8306024 generic-all +gc/stress/systemgc/TestSystemGCWithShenandoah.java#generational 8306024 generic-all +gc/stress/gclocker/TestGCLockerWithShenandoah.java#generational 8306024 generic-all +gc/TestAllocHumongousFragment#generational 8306024 generic-all +java/lang/Thread/virtual/TraceVirtualThreadLocals 8306024 generic-all + ############################################################################# # :hotspot_runtime From 42da31cd9384090f5c5565f371fbca01edb52d53 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Tue, 18 Apr 2023 16:59:44 +0000 Subject: [PATCH 217/254] Disable verification of adjusted capacity Reviewed-by: ysr, kdnilsen, shade --- .../gc/shenandoah/shenandoahGeneration.cpp | 20 ++++++++++--------- .../gc/shenandoah/shenandoahVerifier.cpp | 11 +++++----- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index 9b1569bf858c4..24648bc22cd2a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -441,15 +441,17 @@ void ShenandoahGeneration::adjust_evacuation_budgets(ShenandoahHeap* heap, Shena size_t loaned_regions = 0; size_t available_loan_remnant = 0; // loaned memory that is not yet dedicated to any particular budget - assert(((consumed_by_advance_promotion * 33) / 32) >= collection_set->get_young_bytes_to_be_promoted() * ShenandoahEvacWaste, - "Advance promotion (" SIZE_FORMAT ") should be at least young_bytes_to_be_promoted (" SIZE_FORMAT - ")* ShenandoahEvacWaste, totalling: " SIZE_FORMAT ", within round-off errors of up to 3.125%%", - consumed_by_advance_promotion, collection_set->get_young_bytes_to_be_promoted(), - (size_t) (collection_set->get_young_bytes_to_be_promoted() * ShenandoahEvacWaste)); - - assert(consumed_by_advance_promotion <= (collection_set->get_young_bytes_to_be_promoted() * ShenandoahEvacWaste * 33) / 32, - "Round-off errors should be less than 3.125%%, consumed by advance: " SIZE_FORMAT ", promoted: " SIZE_FORMAT, - consumed_by_advance_promotion, (size_t) (collection_set->get_young_bytes_to_be_promoted() * ShenandoahEvacWaste)); + // We expect this code to be replaced by 05/01/23. + // + // assert(((consumed_by_advance_promotion * 33) / 32) >= collection_set->get_young_bytes_to_be_promoted() * ShenandoahEvacWaste, + // "Advance promotion (" SIZE_FORMAT ") should be at least young_bytes_to_be_promoted (" SIZE_FORMAT + // ")* ShenandoahEvacWaste, totalling: " SIZE_FORMAT ", within round-off errors of up to 3.125%%", + // consumed_by_advance_promotion, collection_set->get_young_bytes_to_be_promoted(), + // (size_t) (collection_set->get_young_bytes_to_be_promoted() * ShenandoahEvacWaste)); + + // assert(consumed_by_advance_promotion <= (collection_set->get_young_bytes_to_be_promoted() * ShenandoahEvacWaste * 33) / 32, + // "Round-off errors should be less than 3.125%%, consumed by advance: " SIZE_FORMAT ", promoted: " SIZE_FORMAT, + // consumed_by_advance_promotion, (size_t) (collection_set->get_young_bytes_to_be_promoted() * ShenandoahEvacWaste)); collection_set->abandon_preselected(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp index b6a1203683bec..4e1bdcfa4a447 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp @@ -411,11 +411,12 @@ class ShenandoahGenerationStatsClosure : public ShenandoahHeapRegionClosure { "%s: generation (%s) used regions (" SIZE_FORMAT ") must equal regions that are in use (" SIZE_FORMAT ")", label, generation->name(), generation->used_regions(), stats.regions()); - size_t capacity = generation->adjusted_capacity(); - guarantee(stats.span() <= capacity, - "%s: generation (%s) size spanned by regions (" SIZE_FORMAT ") must not exceed current capacity (" SIZE_FORMAT "%s)", - label, generation->name(), stats.regions(), - byte_size_in_proper_unit(capacity), proper_unit_for_byte_size(capacity)); +// This check is disabled because of known issues with this feature. We expect this code to be updated by 05/2023. +// size_t capacity = generation->adjusted_capacity(); +// guarantee(stats.span() <= capacity, +// "%s: generation (%s) size spanned by regions (" SIZE_FORMAT ") must not exceed current capacity (" SIZE_FORMAT "%s)", +// label, generation->name(), stats.regions(), +// byte_size_in_proper_unit(capacity), proper_unit_for_byte_size(capacity)); } }; From 79bbb5abe1389c179ae1462649107dc83e40ee36 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Tue, 18 Apr 2023 17:06:17 +0000 Subject: [PATCH 218/254] Account for humongous object waste Reviewed-by: ysr, kdnilsen --- .../share/gc/shenandoah/shenandoahFreeSet.cpp | 20 ++++++--- .../share/gc/shenandoah/shenandoahFreeSet.hpp | 1 + .../share/gc/shenandoah/shenandoahFullGC.cpp | 6 +-- .../gc/shenandoah/shenandoahGeneration.cpp | 45 ++++++++++++------- .../gc/shenandoah/shenandoahGeneration.hpp | 12 +++++ .../gc/shenandoah/shenandoahHeapRegion.cpp | 34 +++++++++++--- .../gc/shenandoah/shenandoahHeapRegion.hpp | 1 + .../gc/shenandoah/shenandoahVerifier.cpp | 16 +++++-- 8 files changed, 102 insertions(+), 33 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index 040527033cecd..0f0b2679879ce 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -579,13 +579,23 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req) { PTR_FORMAT " at transition from FREE to %s", p2i(r->bottom()), p2i(r->end()), p2i(ctx->top_bitmap(r)), req.affiliation_name()); + // While individual regions report their true use, all humongous regions are marked used in the free set. _mutator_free_bitmap.clear_bit(r->index()); } - - // While individual regions report their true use, all humongous regions are - // marked used in the free set. - increase_used(ShenandoahHeapRegion::region_size_bytes() * num); - generation->increase_used(words_size * HeapWordSize); + size_t total_humongous_size = ShenandoahHeapRegion::region_size_bytes() * num; + increase_used(total_humongous_size); + if (_heap->mode()->is_generational()) { + size_t humongous_waste = total_humongous_size - words_size * HeapWordSize; + _heap->global_generation()->increase_used(words_size * HeapWordSize); + _heap->global_generation()->increase_humongous_waste(humongous_waste); + if (req.is_young()) { + _heap->young_generation()->increase_used(words_size * HeapWordSize); + _heap->young_generation()->increase_humongous_waste(humongous_waste); + } else if (req.is_old()) { + _heap->old_generation()->increase_used(words_size * HeapWordSize); + _heap->old_generation()->increase_humongous_waste(humongous_waste); + } + } if (remainder != 0) { // Record this remainder as allocation waste diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp index 0e744d4e7021f..188163729ec77 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp @@ -68,6 +68,7 @@ class ShenandoahFreeSet : public CHeapObj { void adjust_bounds(); bool touches_bounds(size_t num) const; + // Used of free set represents the amount of is_mutator_free set that has been consumed since most recent rebuild. void increase_used(size_t amount); void clear_internal(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index 2da28e5534ed0..7ac06dea01e40 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -202,8 +202,7 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) { // Defer unadjust_available() invocations until after Full GC finishes its efforts because Full GC makes use // of young-gen memory that may have been loaned from old-gen. - // No need to old_gen->increase_used(). That was done when plabs were allocated, accounting for both old evacs and promotions. - + // No need for old_gen->increase_used() as this was done when plabs were allocated. heap->set_alloc_supplement_reserve(0); heap->set_young_evac_reserve(0); heap->set_old_evac_reserve(0); @@ -1502,10 +1501,9 @@ void ShenandoahFullGC::phase5_epilog() { byte_size_in_proper_unit(heap->young_generation()->used()), proper_unit_for_byte_size(heap->young_generation()->used()), byte_size_in_proper_unit(heap->old_generation()->used()), proper_unit_for_byte_size(heap->old_generation()->used())); } - heap->collection_set()->clear(); heap->free_set()->rebuild(); } - heap->clear_cancelled_gc(true /* clear oom handler */); } + diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index 24648bc22cd2a..bb71625d94187 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -162,17 +162,16 @@ void ShenandoahGeneration::log_status(const char *msg) const { size_t v_soft_max_capacity = soft_max_capacity(); size_t v_max_capacity = max_capacity(); size_t v_available = available(); - size_t v_adjusted_avail = adjusted_available(); + size_t v_humongous_waste = get_humongous_waste(); LogGcInfo::print("%s: %s generation used: " SIZE_FORMAT "%s, used regions: " SIZE_FORMAT "%s, " - "soft capacity: " SIZE_FORMAT "%s, max capacity: " SIZE_FORMAT "%s, available: " SIZE_FORMAT "%s, " - "adjusted available: " SIZE_FORMAT "%s", - msg, name(), + "humongous waste: " SIZE_FORMAT "%s, soft capacity: " SIZE_FORMAT "%s, max capacity: " SIZE_FORMAT "%s, " + "available: " SIZE_FORMAT "%s", msg, name(), byte_size_in_proper_unit(v_used), proper_unit_for_byte_size(v_used), byte_size_in_proper_unit(v_used_regions), proper_unit_for_byte_size(v_used_regions), + byte_size_in_proper_unit(v_humongous_waste), proper_unit_for_byte_size(v_humongous_waste), byte_size_in_proper_unit(v_soft_max_capacity), proper_unit_for_byte_size(v_soft_max_capacity), byte_size_in_proper_unit(v_max_capacity), proper_unit_for_byte_size(v_max_capacity), - byte_size_in_proper_unit(v_available), proper_unit_for_byte_size(v_available), - byte_size_in_proper_unit(v_adjusted_avail), proper_unit_for_byte_size(v_adjusted_avail)); + byte_size_in_proper_unit(v_available), proper_unit_for_byte_size(v_available)); } void ShenandoahGeneration::reset_mark_bitmap() { @@ -893,7 +892,7 @@ ShenandoahGeneration::ShenandoahGeneration(ShenandoahGenerationType type, _task_queues(new ShenandoahObjToScanQueueSet(max_workers)), _ref_processor(new ShenandoahReferenceProcessor(MAX2(max_workers, 1U))), _collection_thread_time_s(0.0), - _affiliated_region_count(0), _used(0), _bytes_allocated_since_gc_start(0), + _affiliated_region_count(0), _humongous_waste(0), _used(0), _bytes_allocated_since_gc_start(0), _max_capacity(max_capacity), _soft_max_capacity(soft_max_capacity), _adjusted_capacity(soft_max_capacity), _heuristics(nullptr) { _is_marking_complete.set(); @@ -957,13 +956,15 @@ size_t ShenandoahGeneration::decrement_affiliated_region_count() { } void ShenandoahGeneration::establish_usage(size_t num_regions, size_t num_bytes, size_t humongous_waste) { + assert(ShenandoahHeap::heap()->mode()->is_generational(), "Only generational mode accounts for generational usage"); assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at a safepoint"); _affiliated_region_count = num_regions; _used = num_bytes; - // future improvement: _humongous_waste = humongous_waste; + _humongous_waste = humongous_waste; } void ShenandoahGeneration::clear_used() { + assert(ShenandoahHeap::heap()->mode()->is_generational(), "Only generational mode accounts for generational usage"); assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at a safepoint"); // Do this atomically to assure visibility to other threads, even though these other threads may be idle "right now".. Atomic::store(&_used, (size_t)0); @@ -978,6 +979,23 @@ void ShenandoahGeneration::decrease_used(size_t bytes) { Atomic::sub(&_used, bytes); } +void ShenandoahGeneration::increase_humongous_waste(size_t bytes) { + if (bytes > 0) { + shenandoah_assert_heaplocked_or_fullgc_safepoint(); + _humongous_waste += bytes; + } +} + +void ShenandoahGeneration::decrease_humongous_waste(size_t bytes) { + if (bytes > 0) { + shenandoah_assert_heaplocked_or_fullgc_safepoint(); + assert(_humongous_waste >= bytes, "Waste cannot be negative"); + assert(ShenandoahHeap::heap()->is_full_gc_in_progress() || (_humongous_waste >= bytes), + "Waste (" SIZE_FORMAT ") cannot be negative (after subtracting " SIZE_FORMAT ")", _humongous_waste, bytes); + _humongous_waste -= bytes; + } +} + size_t ShenandoahGeneration::used_regions() const { return _affiliated_region_count; } @@ -997,19 +1015,14 @@ size_t ShenandoahGeneration::used_regions_size() const { } size_t ShenandoahGeneration::available() const { - size_t in_use = used(); + size_t in_use = used() + get_humongous_waste(); size_t soft_capacity = soft_max_capacity(); return in_use > soft_capacity ? 0 : soft_capacity - in_use; } size_t ShenandoahGeneration::adjust_available(intptr_t adjustment) { - // TODO: ysr: remove this check & warning - if (adjustment % ShenandoahHeapRegion::region_size_bytes() != 0) { - log_warning(gc)("Adjustment (" INTPTR_FORMAT ") should be a multiple of region size (" SIZE_FORMAT ")", - adjustment, ShenandoahHeapRegion::region_size_bytes()); - } assert(adjustment % ShenandoahHeapRegion::region_size_bytes() == 0, - "Adjustment to generation size must be multiple of region size"); + "Adjustment to generation size must be multiple of region size"); _adjusted_capacity = soft_max_capacity() + adjustment; return _adjusted_capacity; } @@ -1020,7 +1033,7 @@ size_t ShenandoahGeneration::unadjust_available() { } size_t ShenandoahGeneration::adjusted_available() const { - size_t in_use = used(); + size_t in_use = used() + get_humongous_waste(); size_t capacity = _adjusted_capacity; return in_use > capacity ? 0 : capacity - in_use; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index c3bdc924a9d1c..022090f4c3333 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -52,6 +52,14 @@ class ShenandoahGeneration : public CHeapObj { protected: // Usage size_t _affiliated_region_count; + + // How much free memory is left in the last region of humongous objects. + // This is _not_ included in used, but it _is_ deducted from available, + // which gives the heuristics a more accurate view of how much memory remains + // for allocation. This figure is also included the heap status logging. + // The units are bytes. The value is only changed on a safepoint or under the + // heap lock. + size_t _humongous_waste; volatile size_t _used; volatile size_t _bytes_allocated_since_gc_start; size_t _max_capacity; @@ -196,6 +204,10 @@ class ShenandoahGeneration : public CHeapObj { void increase_used(size_t bytes); void decrease_used(size_t bytes); + void increase_humongous_waste(size_t bytes); + void decrease_humongous_waste(size_t bytes); + size_t get_humongous_waste() const { return _humongous_waste; } + virtual bool is_concurrent_mark_in_progress() = 0; void confirm_heuristics_mode(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index 1385ffca97f7f..a3afa6d2b83e1 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -282,11 +282,17 @@ void ShenandoahHeapRegion::make_trash() { shenandoah_assert_heaplocked(); reset_age(); switch (_state) { - case _cset: - // Reclaiming cset regions case _humongous_start: case _humongous_cont: - // Reclaiming humongous regions + { + // Reclaiming humongous regions and reclaim humongous waste. When this region is eventually recycled, we'll reclaim + // its used memory. At recycle time, we no longer recognize this as a humongous region. + if (ShenandoahHeap::heap()->mode()->is_generational()) { + decrement_humongous_waste(); + } + } + case _cset: + // Reclaiming cset regions case _regular: // Immediate region reclaim set_state(_trash); @@ -655,7 +661,9 @@ void ShenandoahHeapRegion::recycle() { ShenandoahHeap* heap = ShenandoahHeap::heap(); shenandoah_assert_heaplocked(); - heap->generation_for(affiliation())->decrease_used(used()); + if (ShenandoahHeap::heap()->mode()->is_generational()) { + heap->generation_for(affiliation())->decrease_used(used()); + } set_top(bottom()); clear_live_data(); @@ -1036,9 +1044,16 @@ size_t ShenandoahHeapRegion::promote_humongous() { log_debug(gc)("promoting humongous region " SIZE_FORMAT ", from " PTR_FORMAT " to " PTR_FORMAT, r->index(), p2i(r->bottom()), p2i(r->top())); // We mark the entire humongous object's range as dirty after loop terminates, so no need to dirty the range here - r->set_affiliation(OLD_GENERATION); old_generation->increase_used(r->used()); young_generation->decrease_used(r->used()); + r->set_affiliation(OLD_GENERATION); + } + + ShenandoahHeapRegion* tail = heap->get_region(index_limit - 1); + size_t waste = tail->free(); + if (waste != 0) { + old_generation->increase_humongous_waste(waste); + young_generation->decrease_humongous_waste(waste); } // Then fall through to finish the promotion after releasing the heap lock. } else { @@ -1073,3 +1088,12 @@ size_t ShenandoahHeapRegion::promote_humongous() { } return index_limit - index(); } + +void ShenandoahHeapRegion::decrement_humongous_waste() const { + assert(is_humongous(), "Should only use this for humongous regions"); + size_t waste_bytes = free(); + if (waste_bytes > 0) { + ShenandoahHeap::heap()->generation_for(affiliation())->decrease_humongous_waste(waste_bytes); + ShenandoahHeap::heap()->global_generation()->decrease_humongous_waste(waste_bytes); + } +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp index a422f7e25c5d0..269f0fd239be2 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp @@ -461,6 +461,7 @@ class ShenandoahHeapRegion { size_t promote_humongous(); private: + void decrement_humongous_waste() const; void do_commit(); void do_uncommit(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp index 4e1bdcfa4a447..03d53c6f8fa32 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp @@ -347,14 +347,17 @@ class ShenandoahVerifyOopClosure : public BasicOopIterateClosure { // a subset (e.g. the young generation or old generation) of the total heap. class ShenandoahCalculateRegionStatsClosure : public ShenandoahHeapRegionClosure { private: - size_t _used, _committed, _garbage, _regions; + size_t _used, _committed, _garbage, _regions, _humongous_waste; public: - ShenandoahCalculateRegionStatsClosure() : _used(0), _committed(0), _garbage(0), _regions(0) {}; + ShenandoahCalculateRegionStatsClosure() : _used(0), _committed(0), _garbage(0), _regions(0), _humongous_waste(0) {}; void heap_region_do(ShenandoahHeapRegion* r) { _used += r->used(); _garbage += r->garbage(); _committed += r->is_committed() ? ShenandoahHeapRegion::region_size_bytes() : 0; + if (r->is_humongous()) { + _humongous_waste += r->free(); + } _regions++; log_debug(gc)("ShenandoahCalculateRegionStatsClosure: adding " SIZE_FORMAT " for %s Region " SIZE_FORMAT ", yielding: " SIZE_FORMAT, r->used(), (r->is_humongous() ? "humongous" : "regular"), r->index(), _used); @@ -364,6 +367,7 @@ class ShenandoahCalculateRegionStatsClosure : public ShenandoahHeapRegionClosure size_t committed() { return _committed; } size_t garbage() { return _garbage; } size_t regions() { return _regions; } + size_t waste() { return _humongous_waste; } // span is the total memory affiliated with these stats (some of which is in use and other is available) size_t span() { return _regions * ShenandoahHeapRegion::region_size_bytes(); } @@ -402,7 +406,7 @@ class ShenandoahGenerationStatsClosure : public ShenandoahHeapRegionClosure { static void validate_usage(const char* label, ShenandoahGeneration* generation, ShenandoahCalculateRegionStatsClosure& stats) { size_t generation_used = generation->used(); guarantee(stats.used() == generation_used, - "%s: generation (%s) used size must be consistent: generation-used = " SIZE_FORMAT "%s, regions-used = " SIZE_FORMAT "%s", + "%s: generation (%s) used size must be consistent: generation-used: " SIZE_FORMAT "%s, regions-used: " SIZE_FORMAT "%s", label, generation->name(), byte_size_in_proper_unit(generation_used), proper_unit_for_byte_size(generation_used), byte_size_in_proper_unit(stats.used()), proper_unit_for_byte_size(stats.used())); @@ -418,6 +422,12 @@ class ShenandoahGenerationStatsClosure : public ShenandoahHeapRegionClosure { // label, generation->name(), stats.regions(), // byte_size_in_proper_unit(capacity), proper_unit_for_byte_size(capacity)); + size_t humongous_waste = generation->get_humongous_waste(); + guarantee(stats.waste() == humongous_waste, + "%s: generation (%s) humongous waste must be consistent: generation: " SIZE_FORMAT "%s, regions: " SIZE_FORMAT "%s", + label, generation->name(), + byte_size_in_proper_unit(humongous_waste), proper_unit_for_byte_size(humongous_waste), + byte_size_in_proper_unit(stats.waste()), proper_unit_for_byte_size(stats.waste())); } }; From 016bf071e9c8a6d9a2335f8de6ae4a5f49974ea8 Mon Sep 17 00:00:00 2001 From: Aleksey Shipilev Date: Tue, 18 Apr 2023 18:11:29 +0000 Subject: [PATCH 219/254] Specialize ShenandoahGenerationType to avoid generational check penalty on fast paths Reviewed-by: kdnilsen, wkemper --- .../shenandoah/shenandoahConcurrentMark.cpp | 32 +++++++--- .../gc/shenandoah/shenandoahControlThread.cpp | 44 ++++++++----- .../gc/shenandoah/shenandoahControlThread.hpp | 2 + .../share/gc/shenandoah/shenandoahFullGC.cpp | 2 +- .../gc/shenandoah/shenandoahGeneration.hpp | 2 +- .../shenandoah/shenandoahGenerationType.hpp | 11 ++-- .../shenandoah/shenandoahGlobalGeneration.hpp | 4 +- .../share/gc/shenandoah/shenandoahHeap.cpp | 62 +++++++++++-------- .../share/gc/shenandoah/shenandoahHeap.hpp | 4 +- .../shenandoahHeapRegionCounters.cpp | 10 ++- .../share/gc/shenandoah/shenandoahMark.cpp | 7 ++- .../share/gc/shenandoah/shenandoahMark.hpp | 2 +- .../gc/shenandoah/shenandoahMark.inline.hpp | 38 ++++++------ .../shenandoah/shenandoahRegulatorThread.cpp | 4 +- .../share/gc/shenandoah/shenandoahSTWMark.cpp | 9 ++- .../share/gc/shenandoah/shenandoahUtils.hpp | 7 +-- 16 files changed, 148 insertions(+), 92 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp index d14c6efe6dd59..c560131c6d787 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp @@ -198,10 +198,17 @@ void ShenandoahConcurrentMark::mark_concurrent_roots() { workers->run_task(&task); break; } - case GLOBAL: { + case GLOBAL_GEN: { assert(old_task_queues() == nullptr, "Global mark should not have old gen mark queues"); - ShenandoahMarkConcurrentRootsTask task(task_queues(), nullptr, rp, - ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers()); + ShenandoahMarkConcurrentRootsTask task(task_queues(), nullptr, rp, + ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers()); + workers->run_task(&task); + break; + } + case GLOBAL_NON_GEN: { + assert(old_task_queues() == nullptr, "Non-generational mark should not have old gen mark queues"); + ShenandoahMarkConcurrentRootsTask task(task_queues(), nullptr, rp, + ShenandoahPhaseTimings::conc_mark_roots, workers->active_workers()); workers->run_task(&task); break; } @@ -250,9 +257,15 @@ void ShenandoahConcurrentMark::concurrent_mark() { workers->run_task(&task); break; } - case GLOBAL: { + case GLOBAL_GEN: { + TaskTerminator terminator(nworkers, task_queues()); + ShenandoahConcurrentMarkingTask task(this, &terminator); + workers->run_task(&task); + break; + } + case GLOBAL_NON_GEN: { TaskTerminator terminator(nworkers, task_queues()); - ShenandoahConcurrentMarkingTask task(this, &terminator); + ShenandoahConcurrentMarkingTask task(this, &terminator); workers->run_task(&task); break; } @@ -318,8 +331,13 @@ void ShenandoahConcurrentMark::finish_mark_work() { heap->workers()->run_task(&task); break; } - case GLOBAL:{ - ShenandoahFinalMarkingTask task(this, &terminator, ShenandoahStringDedup::is_enabled()); + case GLOBAL_GEN:{ + ShenandoahFinalMarkingTask task(this, &terminator, ShenandoahStringDedup::is_enabled()); + heap->workers()->run_task(&task); + break; + } + case GLOBAL_NON_GEN:{ + ShenandoahFinalMarkingTask task(this, &terminator, ShenandoahStringDedup::is_enabled()); heap->workers()->run_task(&task); break; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index b10e4b98058cd..47ea0574fc15b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -33,6 +33,7 @@ #include "gc/shenandoah/shenandoahFreeSet.hpp" #include "gc/shenandoah/shenandoahFullGC.hpp" #include "gc/shenandoah/shenandoahGeneration.hpp" +#include "gc/shenandoah/shenandoahGlobalGeneration.hpp" #include "gc/shenandoah/shenandoahYoungGeneration.hpp" #include "gc/shenandoah/shenandoahOldGeneration.hpp" #include "gc/shenandoah/shenandoahPhaseTimings.hpp" @@ -61,7 +62,7 @@ ShenandoahControlThread::ShenandoahControlThread() : _regulator_lock(Mutex::nosafepoint - 2, "ShenandoahRegulatorGC_lock", true), _periodic_task(this), _requested_gc_cause(GCCause::_no_cause_specified), - _requested_generation(ShenandoahGenerationType::GLOBAL), + _requested_generation(select_global_generation()), _degen_point(ShenandoahGC::_degenerated_outside_cycle), _degen_generation(nullptr), _allocs_seen(0), @@ -93,7 +94,7 @@ void ShenandoahControlThread::run_service() { ShenandoahHeap* heap = ShenandoahHeap::heap(); GCMode default_mode = concurrent_normal; - ShenandoahGenerationType generation = GLOBAL; + ShenandoahGenerationType generation = select_global_generation(); GCCause::Cause default_cause = GCCause::_shenandoah_concurrent_gc; double last_shrink_time = os::elapsedTime(); @@ -160,12 +161,12 @@ void ShenandoahControlThread::run_service() { } else { heuristics->record_allocation_failure_gc(); policy->record_alloc_failure_to_full(); - generation = GLOBAL; + generation = select_global_generation(); set_gc_mode(stw_full); } } else if (explicit_gc_requested) { cause = requested_gc_cause; - generation = GLOBAL; + generation = select_global_generation(); log_info(gc)("Trigger: Explicit GC request (%s)", GCCause::to_string(cause)); global_heuristics->record_requested_gc(); @@ -181,7 +182,7 @@ void ShenandoahControlThread::run_service() { } } else if (implicit_gc_requested) { cause = requested_gc_cause; - generation = GLOBAL; + generation = select_global_generation(); log_info(gc)("Trigger: Implicit GC request (%s)", GCCause::to_string(cause)); global_heuristics->record_requested_gc(); @@ -219,7 +220,7 @@ void ShenandoahControlThread::run_service() { set_gc_mode(servicing_old); } - if (generation == GLOBAL) { + if (generation == select_global_generation()) { heap->set_unload_classes(global_heuristics->should_unload_classes()); } else { heap->set_unload_classes(false); @@ -245,7 +246,7 @@ void ShenandoahControlThread::run_service() { // Blow all soft references on this cycle, if handling allocation failure, // either implicit or explicit GC request, or we are requested to do so unconditionally. - if (generation == GLOBAL && (alloc_failure_pending || implicit_gc_requested || explicit_gc_requested || ShenandoahAlwaysClearSoftRefs)) { + if (generation == select_global_generation() && (alloc_failure_pending || implicit_gc_requested || explicit_gc_requested || ShenandoahAlwaysClearSoftRefs)) { heap->soft_ref_policy()->set_should_clear_all_soft_refs(true); } @@ -291,7 +292,7 @@ void ShenandoahControlThread::run_service() { heap->set_aging_cycle(was_aging_cycle); if (!service_stw_degenerated_cycle(cause, degen_point)) { // The degenerated GC was upgraded to a Full GC - generation = GLOBAL; + generation = select_global_generation(); } break; } @@ -348,7 +349,7 @@ void ShenandoahControlThread::run_service() { // Clear metaspace oom flag, if current cycle unloaded classes if (heap->unload_classes()) { - assert(generation == GLOBAL, "Only unload classes during GLOBAL cycle"); + assert(generation == select_global_generation(), "Only unload classes during GLOBAL cycle"); global_heuristics->clear_metaspace_oom(); } @@ -471,14 +472,19 @@ void ShenandoahControlThread::service_concurrent_normal_cycle(const ShenandoahHe service_concurrent_cycle(heap->young_generation(), cause, false); break; } - case GLOBAL: { + case OLD: { + log_info(gc, ergo)("Start GC cycle (OLD)"); + service_concurrent_old_cycle(heap, cause); + break; + } + case GLOBAL_GEN: { log_info(gc, ergo)("Start GC cycle (GLOBAL)"); service_concurrent_cycle(heap->global_generation(), cause, false); break; } - case OLD: { - log_info(gc, ergo)("Start GC cycle (OLD)"); - service_concurrent_old_cycle(heap, cause); + case GLOBAL_NON_GEN: { + log_info(gc, ergo)("Start GC cycle"); + service_concurrent_cycle(heap->global_generation(), cause, false); break; } default: @@ -488,10 +494,10 @@ void ShenandoahControlThread::service_concurrent_normal_cycle(const ShenandoahHe if (heap->mode()->is_generational()) { if (heap->cancelled_gc()) { msg = (generation == YOUNG) ? "At end of Interrupted Concurrent Young GC" : - "At end of Interrupted Concurrent Bootstrap GC"; + "At end of Interrupted Concurrent Bootstrap GC"; } else { msg = (generation == YOUNG) ? "At end of Concurrent Young GC" : - "At end of Concurrent Bootstrap GC"; + "At end of Concurrent Bootstrap GC"; } } else { msg = heap->cancelled_gc() ? "At end of cancelled GC" : @@ -1022,3 +1028,11 @@ void ShenandoahControlThread::set_gc_mode(ShenandoahControlThread::GCMode new_mo ml.notify_all(); } } + +ShenandoahGenerationType ShenandoahControlThread::select_global_generation() { + if (ShenandoahHeap::heap()->mode()->is_generational()) { + return GLOBAL_GEN; + } else { + return GLOBAL_NON_GEN; + } +} diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp index 7c937286d53ac..ad16e8c80d253 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp @@ -184,6 +184,8 @@ class ShenandoahControlThread: public ConcurrentGCThread { return _mode; } + static ShenandoahGenerationType select_global_generation(); + private: static const char* gc_mode_name(GCMode mode); void notify_control_thread(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index 7ac06dea01e40..5180ecc47579f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -36,7 +36,7 @@ #include "gc/shenandoah/shenandoahCollectionSet.hpp" #include "gc/shenandoah/shenandoahFreeSet.hpp" #include "gc/shenandoah/shenandoahFullGC.hpp" -#include "gc/shenandoah/shenandoahGeneration.hpp" +#include "gc/shenandoah/shenandoahGlobalGeneration.hpp" #include "gc/shenandoah/shenandoahPhaseTimings.hpp" #include "gc/shenandoah/shenandoahMark.inline.hpp" #include "gc/shenandoah/shenandoahMonitoringSupport.hpp" diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index 022090f4c3333..9e9385e5fa147 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -90,7 +90,7 @@ class ShenandoahGeneration : public CHeapObj { bool is_young() const { return _type == YOUNG; } bool is_old() const { return _type == OLD; } - bool is_global() const { return _type == GLOBAL; } + bool is_global() const { return _type == GLOBAL_GEN || _type == GLOBAL_NON_GEN; } inline ShenandoahGenerationType type() const { return _type; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGenerationType.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGenerationType.hpp index abf1b0b0f42dd..ec73fc07c1378 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGenerationType.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGenerationType.hpp @@ -26,14 +26,17 @@ #define SHARE_GC_SHENANDOAH_SHENANDOAHGENERATIONTYPE_HPP enum ShenandoahGenerationType { - YOUNG, - OLD, - GLOBAL + GLOBAL_NON_GEN, // Global, non-generational + GLOBAL_GEN, // Global, generational + YOUNG, // Young, generational + OLD // Old, generational }; inline const char* shenandoah_generation_name(ShenandoahGenerationType mode) { switch (mode) { - case GLOBAL: + case GLOBAL_NON_GEN: + return ""; + case GLOBAL_GEN: return "Global"; case OLD: return "Old"; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp index c7d6d8a6627f0..0d94ce56ad38d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp @@ -32,8 +32,8 @@ // A "generation" that represents the whole heap. class ShenandoahGlobalGeneration : public ShenandoahGeneration { public: - ShenandoahGlobalGeneration(uint max_queues, size_t max_capacity, size_t soft_max_capacity) - : ShenandoahGeneration(GLOBAL, max_queues, max_capacity, soft_max_capacity) { } + ShenandoahGlobalGeneration(bool generational, uint max_queues, size_t max_capacity, size_t soft_max_capacity) + : ShenandoahGeneration(generational ? GLOBAL_GEN : GLOBAL_NON_GEN, max_queues, max_capacity, soft_max_capacity) { } public: virtual const char* name() const override; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 853e697bbf48e..69d3c8233698f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -187,8 +187,7 @@ jint ShenandoahHeap::initialize() { _committed = _initial_size; // Now we know the number of regions and heap sizes, initialize the heuristics. - initialize_generations(); - initialize_heuristics(); + initialize_heuristics_generations(); size_t heap_page_size = UseLargePages ? os::large_page_size() : os::vm_page_size(); size_t bitmap_page_size = UseLargePages ? os::large_page_size() : os::vm_page_size(); @@ -451,9 +450,13 @@ jint ShenandoahHeap::initialize() { size_t ShenandoahHeap::max_size_for(ShenandoahGeneration* generation) const { switch (generation->type()) { - case YOUNG: return _generation_sizer.max_young_size(); - case OLD: return max_capacity() - _generation_sizer.min_young_size(); - case GLOBAL: return max_capacity(); + case YOUNG: + return _generation_sizer.max_young_size(); + case OLD: + return max_capacity() - _generation_sizer.min_young_size(); + case GLOBAL_GEN: + case GLOBAL_NON_GEN: + return max_capacity(); default: ShouldNotReachHere(); return 0; @@ -462,32 +465,20 @@ size_t ShenandoahHeap::max_size_for(ShenandoahGeneration* generation) const { size_t ShenandoahHeap::min_size_for(ShenandoahGeneration* generation) const { switch (generation->type()) { - case YOUNG: return _generation_sizer.min_young_size(); - case OLD: return max_capacity() - _generation_sizer.max_young_size(); - case GLOBAL: return min_capacity(); + case YOUNG: + return _generation_sizer.min_young_size(); + case OLD: + return max_capacity() - _generation_sizer.max_young_size(); + case GLOBAL_GEN: + case GLOBAL_NON_GEN: + return min_capacity(); default: ShouldNotReachHere(); return 0; } } -void ShenandoahHeap::initialize_generations() { - // Max capacity is the maximum _allowed_ capacity. That is, the maximum allowed capacity - // for old would be total heap - minimum capacity of young. This means the sum of the maximum - // allowed for old and young could exceed the total heap size. It remains the case that the - // _actual_ capacity of young + old = total. - _generation_sizer.heap_size_changed(soft_max_capacity()); - size_t initial_capacity_young = _generation_sizer.max_young_size(); - size_t max_capacity_young = _generation_sizer.max_young_size(); - size_t initial_capacity_old = max_capacity() - max_capacity_young; - size_t max_capacity_old = max_capacity() - initial_capacity_young; - - _young_generation = new ShenandoahYoungGeneration(_max_workers, max_capacity_young, initial_capacity_young); - _old_generation = new ShenandoahOldGeneration(_max_workers, max_capacity_old, initial_capacity_old); - _global_generation = new ShenandoahGlobalGeneration(_max_workers, soft_max_capacity(), soft_max_capacity()); -} - -void ShenandoahHeap::initialize_heuristics() { +void ShenandoahHeap::initialize_heuristics_generations() { if (ShenandoahGCMode != nullptr) { if (strcmp(ShenandoahGCMode, "satb") == 0) { _gc_mode = new ShenandoahSATBMode(); @@ -515,6 +506,20 @@ void ShenandoahHeap::initialize_heuristics() { _gc_mode->name())); } + // Max capacity is the maximum _allowed_ capacity. That is, the maximum allowed capacity + // for old would be total heap - minimum capacity of young. This means the sum of the maximum + // allowed for old and young could exceed the total heap size. It remains the case that the + // _actual_ capacity of young + old = total. + _generation_sizer.heap_size_changed(soft_max_capacity()); + size_t initial_capacity_young = _generation_sizer.max_young_size(); + size_t max_capacity_young = _generation_sizer.max_young_size(); + size_t initial_capacity_old = max_capacity() - max_capacity_young; + size_t max_capacity_old = max_capacity() - initial_capacity_young; + + _young_generation = new ShenandoahYoungGeneration(_max_workers, max_capacity_young, initial_capacity_young); + _old_generation = new ShenandoahOldGeneration(_max_workers, max_capacity_old, initial_capacity_old); + _global_generation = new ShenandoahGlobalGeneration(_gc_mode->is_generational(), _max_workers, soft_max_capacity(), soft_max_capacity()); + _global_generation->initialize_heuristics(_gc_mode); if (mode()->is_generational()) { _young_generation->initialize_heuristics(_gc_mode); @@ -3129,7 +3134,12 @@ void ShenandoahGenerationRegionClosure::heap_region_do(ShenandoahHeapRegion } template<> -void ShenandoahGenerationRegionClosure::heap_region_do(ShenandoahHeapRegion* region) { +void ShenandoahGenerationRegionClosure::heap_region_do(ShenandoahHeapRegion* region) { + _cl->heap_region_do(region); +} + +template<> +void ShenandoahGenerationRegionClosure::heap_region_do(ShenandoahHeapRegion* region) { _cl->heap_region_do(region); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index 31f1c0a08798e..4e0bc45cb7f6b 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -191,9 +191,7 @@ class ShenandoahHeap : public CollectedHeap { ShenandoahHeap(ShenandoahCollectorPolicy* policy); jint initialize() override; void post_initialize() override; - void initialize_mode(); - void initialize_heuristics(); - void initialize_generations(); + void initialize_heuristics_generations(); void initialize_serviceability() override; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp index f7534e2a7a151..66b6ce2bfaf1c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegionCounters.cpp @@ -157,9 +157,13 @@ static int encode_phase(ShenandoahHeap* heap) { static int get_generation_shift(ShenandoahGeneration* generation) { switch (generation->type()) { - case GLOBAL: return 0; - case OLD: return 2; - case YOUNG: return 4; + case GLOBAL_NON_GEN: + case GLOBAL_GEN: + return 0; + case OLD: + return 2; + case YOUNG: + return 4; default: ShouldNotReachHere(); return -1; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp index 7cf3c3ed9f163..483858c368bc8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.cpp @@ -98,8 +98,11 @@ void ShenandoahMark::mark_loop(ShenandoahGenerationType generation, uint worker_ // Old generation collection only performs marking, it should not update references. mark_loop_prework(worker_id, terminator, rp, req, false); break; - case GLOBAL: - mark_loop_prework(worker_id, terminator, rp, req, update_refs); + case GLOBAL_GEN: + mark_loop_prework(worker_id, terminator, rp, req, update_refs); + break; + case GLOBAL_NON_GEN: + mark_loop_prework(worker_id, terminator, rp, req, update_refs); break; default: ShouldNotReachHere(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.hpp index b36d57751b992..8ee5a5d2d4715 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMark.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.hpp @@ -83,7 +83,7 @@ class ShenandoahMark: public StackObj { void mark_loop_prework(uint worker_id, TaskTerminator *terminator, ShenandoahReferenceProcessor *rp, StringDedup::Requests* const req, bool update_refs); template - static bool in_generation(oop obj); + static bool in_generation(ShenandoahHeap* const heap, oop obj); static void mark_ref(ShenandoahObjToScanQueue* q, ShenandoahMarkingContext* const mark_context, diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp index 917a696a673fb..6529f059db225 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMark.inline.hpp @@ -263,13 +263,13 @@ class ShenandoahSATBBufferClosure : public SATBBufferClosure { }; template -bool ShenandoahMark::in_generation(oop obj) { +bool ShenandoahMark::in_generation(ShenandoahHeap* const heap, oop obj) { // Each in-line expansion of in_generation() resolves GENERATION at compile time. if (GENERATION == YOUNG) { - return ShenandoahHeap::heap()->is_in_young(obj); + return heap->is_in_young(obj); } else if (GENERATION == OLD) { - return ShenandoahHeap::heap()->is_in_old(obj); - } else if (GENERATION == GLOBAL) { + return heap->is_in_old(obj); + } else if (GENERATION == GLOBAL_GEN || GENERATION == GLOBAL_NON_GEN) { return true; } else { return false; @@ -278,6 +278,9 @@ bool ShenandoahMark::in_generation(oop obj) { template inline void ShenandoahMark::mark_through_ref(T *p, ShenandoahObjToScanQueue* q, ShenandoahObjToScanQueue* old_q, ShenandoahMarkingContext* const mark_context, bool weak) { + // Note: This is a very hot code path, so the code should be conditional on GENERATION template + // parameter where possible, in order to generate the most efficient code. + T o = RawAccess<>::oop_load(p); if (!CompressedOops::is_null(o)) { oop obj = CompressedOops::decode_not_null(o); @@ -285,23 +288,20 @@ inline void ShenandoahMark::mark_through_ref(T *p, ShenandoahObjToScanQueue* q, ShenandoahHeap* heap = ShenandoahHeap::heap(); shenandoah_assert_not_forwarded(p, obj); shenandoah_assert_not_in_cset_except(p, obj, heap->cancelled_gc()); - if (in_generation(obj)) { + if (in_generation(heap, obj)) { mark_ref(q, mark_context, weak, obj); shenandoah_assert_marked(p, obj); - // TODO: This is v-call on very hot path, can we sense the same from GENERATION? - if (heap->mode()->is_generational()) { - // TODO: As implemented herein, GLOBAL collections reconstruct the card table during GLOBAL concurrent - // marking. Note that the card table is cleaned at init_mark time so it needs to be reconstructed to support - // future young-gen collections. It might be better to reconstruct card table in - // ShenandoahHeapRegion::global_oop_iterate_and_fill_dead. We could either mark all live memory as dirty, or could - // use the GLOBAL update-refs scanning of pointers to determine precisely which cards to flag as dirty. - if (GENERATION == YOUNG && heap->is_in_old(p)) { - // Mark card as dirty because remembered set scanning still finds interesting pointer. - heap->mark_card_as_dirty((HeapWord*)p); - } else if (GENERATION == GLOBAL && heap->is_in_old(p) && heap->is_in_young(obj)) { - // Mark card as dirty because GLOBAL marking finds interesting pointer. - heap->mark_card_as_dirty((HeapWord*)p); - } + // TODO: As implemented herein, GLOBAL_GEN collections reconstruct the card table during GLOBAL_GEN concurrent + // marking. Note that the card table is cleaned at init_mark time so it needs to be reconstructed to support + // future young-gen collections. It might be better to reconstruct card table in + // ShenandoahHeapRegion::global_oop_iterate_and_fill_dead. We could either mark all live memory as dirty, or could + // use the GLOBAL update-refs scanning of pointers to determine precisely which cards to flag as dirty. + if (GENERATION == YOUNG && heap->is_in_old(p)) { + // Mark card as dirty because remembered set scanning still finds interesting pointer. + heap->mark_card_as_dirty((HeapWord*)p); + } else if (GENERATION == GLOBAL_GEN && heap->is_in_old(p) && heap->is_in_young(obj)) { + // Mark card as dirty because GLOBAL marking finds interesting pointer. + heap->mark_card_as_dirty((HeapWord*)p); } } else if (old_q != nullptr) { // Young mark, bootstrapping old_q or concurrent with old_q marking. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.cpp index 2cbbe6b534760..7664b75d1d649 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.cpp @@ -72,7 +72,7 @@ void ShenandoahRegulatorThread::regulate_concurrent_cycles() { ShenandoahControlThread::GCMode mode = _control_thread->gc_mode(); if (mode == ShenandoahControlThread::none) { if (should_unload_classes()) { - if (_control_thread->request_concurrent_gc(GLOBAL)) { + if (_control_thread->request_concurrent_gc(ShenandoahControlThread::select_global_generation())) { log_info(gc)("Heuristics request for global (unload classes) accepted."); } } else { @@ -148,7 +148,7 @@ bool ShenandoahRegulatorThread::start_young_cycle() { } bool ShenandoahRegulatorThread::start_global_cycle() { - return _global_heuristics->should_start_gc() && _control_thread->request_concurrent_gc(GLOBAL); + return _global_heuristics->should_start_gc() && _control_thread->request_concurrent_gc(ShenandoahControlThread::select_global_generation()); } void ShenandoahRegulatorThread::stop_service() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp b/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp index a51a2643edaf9..510d1473b58a7 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.cpp @@ -139,8 +139,13 @@ void ShenandoahSTWMark::mark() { void ShenandoahSTWMark::mark_roots(uint worker_id) { switch (_generation->type()) { - case GLOBAL: { - ShenandoahInitMarkRootsClosure init_mark(task_queues()->queue(worker_id)); + case GLOBAL_NON_GEN: { + ShenandoahInitMarkRootsClosure init_mark(task_queues()->queue(worker_id)); + _root_scanner.roots_do(&init_mark, worker_id); + break; + } + case GLOBAL_GEN: { + ShenandoahInitMarkRootsClosure init_mark(task_queues()->queue(worker_id)); _root_scanner.roots_do(&init_mark, worker_id); break; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp b/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp index c8cdbe2f96341..60ae96077ab83 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp @@ -45,11 +45,10 @@ class GCTimer; class ShenandoahGeneration; #define SHENANDOAH_RETURN_EVENT_MESSAGE(heap, generation_type, prefix, postfix) \ - if (!heap->mode()->is_generational()) { \ - return prefix "" postfix; \ - } \ switch (generation_type) { \ - case GLOBAL: \ + case GLOBAL_NON_GEN: \ + return prefix "" postfix; \ + case GLOBAL_GEN: \ return prefix " (GLOBAL)" postfix; \ case YOUNG: \ return prefix " (YOUNG)" postfix; \ From 5984bd2966c2bf82f394a36f337740fb4669b39b Mon Sep 17 00:00:00 2001 From: William Kemper Date: Wed, 19 Apr 2023 17:15:20 +0000 Subject: [PATCH 220/254] Add unique ticket number for each disabled test Reviewed-by: ysr, shade --- test/hotspot/jtreg/ProblemList.txt | 19 +++++++++---------- test/jdk/ProblemList.txt | 1 + 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/test/hotspot/jtreg/ProblemList.txt b/test/hotspot/jtreg/ProblemList.txt index 62a173f7daffb..3152d01d580b7 100644 --- a/test/hotspot/jtreg/ProblemList.txt +++ b/test/hotspot/jtreg/ProblemList.txt @@ -83,16 +83,15 @@ gc/stress/gclocker/TestGCLockerWithG1.java 8180622 generic-all gc/stress/TestJNIBlockFullGC/TestJNIBlockFullGC.java 8192647 generic-all gc/stress/TestStressG1Humongous.java 8286554 windows-x64 -gc/shenandoah/TestDynamicSoftMaxHeapSize.java#generational 8306024 generic-all -gc/shenandoah/TestArrayCopyStress.java#generational 8306024 generic-all -gc/shenandoah/oom/TestThreadFailure.java 8306024 generic-all -gc/shenandoah/oom/TestClassLoaderLeak 8306024 generic-all -gc/shenandoah/mxbeans/TestChurnNotifications#generational 8306024 generic-all -gc/stress/gcold/TestGCOldWithShenandoah.java#generational 8306024 generic-all -gc/stress/systemgc/TestSystemGCWithShenandoah.java#generational 8306024 generic-all -gc/stress/gclocker/TestGCLockerWithShenandoah.java#generational 8306024 generic-all -gc/TestAllocHumongousFragment#generational 8306024 generic-all -java/lang/Thread/virtual/TraceVirtualThreadLocals 8306024 generic-all +gc/shenandoah/TestDynamicSoftMaxHeapSize.java#generational 8306333 generic-all +gc/shenandoah/TestArrayCopyStress.java#generational 8306334 generic-all +gc/shenandoah/oom/TestThreadFailure.java 8306335 generic-all +gc/shenandoah/oom/TestClassLoaderLeak.java 8306336 generic-all +gc/shenandoah/mxbeans/TestChurnNotifications.java#generational 8306337 generic-all +gc/stress/gcold/TestGCOldWithShenandoah.java#generational 8306339 generic-all +gc/stress/systemgc/TestSystemGCWithShenandoah.java#generational 8306340 generic-all +gc/stress/gclocker/TestGCLockerWithShenandoah.java#generational 8306341 generic-all +gc/TestAllocHumongousFragment.java#generational 8306342 generic-all ############################################################################# diff --git a/test/jdk/ProblemList.txt b/test/jdk/ProblemList.txt index 5ff960b24ae4e..2c9a6351b33c5 100644 --- a/test/jdk/ProblemList.txt +++ b/test/jdk/ProblemList.txt @@ -484,6 +484,7 @@ java/lang/invoke/LFCaching/LFMultiThreadCachingTest.java 8151492 generic- java/lang/invoke/LFCaching/LFGarbageCollectedTest.java 8078602 generic-all java/lang/invoke/lambda/LambdaFileEncodingSerialization.java 8249079 linux-x64 java/lang/invoke/RicochetTest.java 8251969 generic-all +java/lang/Thread/virtual/TraceVirtualThreadLocals.java 8306343 generic-all ############################################################################ From badb7388a35f8a21a9c372954458c6e9e9177923 Mon Sep 17 00:00:00 2001 From: Aleksey Shipilev Date: Fri, 21 Apr 2023 18:51:53 +0000 Subject: [PATCH 221/254] Eliminate TODO in ShenandoahHeuristics::select_aged_regions Reviewed-by: wkemper --- .../share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp index 2a811b591262e..ec0c365b873b1 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp @@ -78,10 +78,7 @@ ShenandoahHeuristics::~ShenandoahHeuristics() { size_t ShenandoahHeuristics::select_aged_regions(size_t old_available, size_t num_regions, bool* preselected_regions) { ShenandoahHeap* heap = ShenandoahHeap::heap(); - if (!heap->mode()->is_generational()) { - // TODO: Do we need this check, or assert is enough? - return 0; - } + assert(heap->mode()->is_generational(), "Only in generational mode"); size_t old_consumed = 0; for (size_t i = 0; i < num_regions; i++) { From 64d14f3bc64c47b226876ad9865c6988c31bea7b Mon Sep 17 00:00:00 2001 From: William Kemper Date: Mon, 24 Apr 2023 17:36:45 +0000 Subject: [PATCH 222/254] 8306334: Handle preemption of old cycle between filling and bootstrap phases Reviewed-by: kdnilsen --- .../gc/shenandoah/shenandoahControlThread.cpp | 29 ++++++++++++++++--- .../gc/shenandoah/shenandoahControlThread.hpp | 1 + .../gc/shenandoah/shenandoahOldGeneration.cpp | 3 +- test/hotspot/jtreg/ProblemList.txt | 1 - 4 files changed, 27 insertions(+), 7 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index 47ea0574fc15b..144d9095b0b3c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -526,11 +526,32 @@ void ShenandoahControlThread::service_concurrent_old_cycle(const ShenandoahHeap* _allow_old_preemption.unset(); if (heap->is_prepare_for_old_mark_in_progress()) { + // Coalescing threads detected the cancellation request and aborted. Stay + // in this state so control thread may resume the coalescing work. assert(old_generation->state() == ShenandoahOldGeneration::FILLING, "Prepare for mark should be in progress"); return; } - assert(old_generation->state() == ShenandoahOldGeneration::BOOTSTRAPPING, "Finished with filling, should be bootstrapping"); + // It is possible for a young generation request to preempt this nascent old + // collection cycle _after_ we've finished making the old regions parseable (filling), + // but _before_ we have unset the preemption flag. It is also possible for an + // allocation failure to occur after the threads have finished filling. We must + // check if we have been cancelled before we start a bootstrap cycle. + if (check_cancellation_or_degen(ShenandoahGC::_degenerated_outside_cycle)) { + if (heap->cancelled_gc()) { + // If this was a preemption request, the cancellation would have been cleared + // so that we run a concurrent young cycle. If the cancellation is still set, + // then this is an allocation failure and we need to run a degenerated cycle. + // If this is a preemption request, we're just going to fall through and run + // the bootstrap cycle to start the old generation cycle (the bootstrap cycle is + // a concurrent young cycle - which is what we're being asked to do in that case). + // If the cycle is cancelled for any other reason, we return from here and let + // the control thread return to the top of its decision loop. + log_info(gc)("Preparation for old generation cycle was cancelled"); + return; + } + } + old_generation->transition_to(ShenandoahOldGeneration::BOOTSTRAPPING); } case ShenandoahOldGeneration::BOOTSTRAPPING: { // Configure the young generation's concurrent mark to put objects in @@ -538,6 +559,7 @@ void ShenandoahControlThread::service_concurrent_old_cycle(const ShenandoahHeap* // generation. The young cycle will run as normal except that rather than // ignore old references it will mark and enqueue them in the old concurrent // task queues but it will not traverse them. + set_gc_mode(bootstrapping_old); young_generation->set_old_gen_task_queues(old_generation->task_queues()); ShenandoahGCSession session(cause, young_generation); service_concurrent_cycle(heap,young_generation, cause, true); @@ -556,9 +578,7 @@ void ShenandoahControlThread::service_concurrent_old_cycle(const ShenandoahHeap* // From here we will 'resume' the old concurrent mark. This will skip reset // and init mark for the concurrent mark. All of that work will have been - // done by the bootstrapping young cycle. In order to simplify the debugging - // effort, the old cycle will ONLY complete the mark phase. No actual - // collection of the old generation is happening here. + // done by the bootstrapping young cycle. set_gc_mode(servicing_old); old_generation->transition_to(ShenandoahOldGeneration::MARKING); } @@ -1016,6 +1036,7 @@ const char* ShenandoahControlThread::gc_mode_name(ShenandoahControlThread::GCMod case stw_degenerated: return "degenerated"; case stw_full: return "full"; case servicing_old: return "old"; + case bootstrapping_old: return "bootstrap"; default: return "unknown"; } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp index ad16e8c80d253..f9596feeeb7bd 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp @@ -74,6 +74,7 @@ class ShenandoahControlThread: public ConcurrentGCThread { concurrent_normal, stw_degenerated, stw_full, + bootstrapping_old, servicing_old } GCMode; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp index 291a129e10078..331499816f815 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp @@ -267,11 +267,10 @@ bool ShenandoahOldGeneration::coalesce_and_fill() { // Remember that we're done with coalesce-and-fill. heap->set_prepare_for_old_mark_in_progress(false); old_heuristics->abandon_collection_candidates(); - transition_to(BOOTSTRAPPING); return true; } else { + // Otherwise, we were preempted before the work was done. log_debug(gc)("Suspending coalesce-and-fill of old heap regions"); - // Otherwise, we got preempted before the work was done. return false; } } diff --git a/test/hotspot/jtreg/ProblemList.txt b/test/hotspot/jtreg/ProblemList.txt index 3152d01d580b7..48e37e6d3b46b 100644 --- a/test/hotspot/jtreg/ProblemList.txt +++ b/test/hotspot/jtreg/ProblemList.txt @@ -84,7 +84,6 @@ gc/stress/TestJNIBlockFullGC/TestJNIBlockFullGC.java 8192647 generic-all gc/stress/TestStressG1Humongous.java 8286554 windows-x64 gc/shenandoah/TestDynamicSoftMaxHeapSize.java#generational 8306333 generic-all -gc/shenandoah/TestArrayCopyStress.java#generational 8306334 generic-all gc/shenandoah/oom/TestThreadFailure.java 8306335 generic-all gc/shenandoah/oom/TestClassLoaderLeak.java 8306336 generic-all gc/shenandoah/mxbeans/TestChurnNotifications.java#generational 8306337 generic-all From d59effcb78d15217a5457e93a40dffa32c68bc65 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Tue, 25 Apr 2023 20:13:49 +0000 Subject: [PATCH 223/254] Use state of mark context itself to track stability of old mark bitmap Reviewed-by: ysr, shade --- src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 69d3c8233698f..e146b486416ee 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -688,9 +688,7 @@ bool ShenandoahHeap::doing_mixed_evacuations() { } bool ShenandoahHeap::is_old_bitmap_stable() const { - ShenandoahOldGeneration::State state = _old_generation->state(); - return state != ShenandoahOldGeneration::MARKING - && state != ShenandoahOldGeneration::BOOTSTRAPPING; + return _old_generation->is_mark_complete(); } bool ShenandoahHeap::is_gc_generation_young() const { From d254fb2f0143c88595f8100f9e5e0bdce83b81ad Mon Sep 17 00:00:00 2001 From: William Kemper Date: Wed, 26 Apr 2023 15:28:16 +0000 Subject: [PATCH 224/254] Usage tracking cleanup Reviewed-by: kdnilsen, ysr --- .../gc/shenandoah/shenandoahAllocRequest.hpp | 47 +++- .../share/gc/shenandoah/shenandoahFreeSet.cpp | 200 ++++++++---------- .../share/gc/shenandoah/shenandoahFreeSet.hpp | 1 + .../share/gc/shenandoah/shenandoahFullGC.cpp | 44 ++-- .../gc/shenandoah/shenandoahGeneration.cpp | 18 +- .../gc/shenandoah/shenandoahGeneration.hpp | 2 +- .../shenandoah/shenandoahGlobalGeneration.cpp | 4 - .../shenandoah/shenandoahGlobalGeneration.hpp | 1 - .../share/gc/shenandoah/shenandoahHeap.cpp | 101 +++++++-- .../share/gc/shenandoah/shenandoahHeap.hpp | 12 +- .../gc/shenandoah/shenandoahHeapRegion.cpp | 17 +- .../shenandoahHeapRegion.inline.hpp | 1 + .../gc/shenandoah/shenandoah_globals.hpp | 3 +- .../gc/shenandoah/vmStructs_shenandoah.hpp | 6 +- .../gc/shenandoah/ShenandoahGeneration.java | 59 ++++++ .../hotspot/gc/shenandoah/ShenandoahHeap.java | 8 +- 16 files changed, 311 insertions(+), 213 deletions(-) create mode 100644 src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/shenandoah/ShenandoahGeneration.java diff --git a/src/hotspot/share/gc/shenandoah/shenandoahAllocRequest.hpp b/src/hotspot/share/gc/shenandoah/shenandoahAllocRequest.hpp index 6f2dc92937109..bd0fbe53dc0a3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahAllocRequest.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahAllocRequest.hpp @@ -59,18 +59,35 @@ class ShenandoahAllocRequest : StackObj { } private: + // When ShenandoahElasticTLAB is enabled, the request cannot be made smaller than _min_size. size_t _min_size; + + // The size of the request in words. size_t _requested_size; + + // The allocation may be increased for padding or decreased to fit in the remaining space of a region. size_t _actual_size; + + // For a humongous object, the _waste is the amount of free memory in the last region. + // For other requests, the _waste will be non-zero if the request enountered one or more regions + // with less memory than _min_size. This waste does not contribute to the used memory for + // the heap, but it does contribute to the allocation rate for heuristics. + size_t _waste; + + // This is the type of the request. Type _alloc_type; + + // This is the generation which the request is targeting. ShenandoahAffiliation const _affiliation; + #ifdef ASSERT + // Check that this is set before being read. bool _actual_size_set; #endif ShenandoahAllocRequest(size_t _min_size, size_t _requested_size, Type _alloc_type, ShenandoahAffiliation affiliation) : _min_size(_min_size), _requested_size(_requested_size), - _actual_size(0), _alloc_type(_alloc_type), _affiliation(affiliation) + _actual_size(0), _waste(0), _alloc_type(_alloc_type), _affiliation(affiliation) #ifdef ASSERT , _actual_size_set(false) #endif @@ -97,24 +114,24 @@ class ShenandoahAllocRequest : StackObj { return ShenandoahAllocRequest(0, requested_size, _alloc_shared, ShenandoahAffiliation::YOUNG_GENERATION); } - inline size_t size() { + inline size_t size() const { return _requested_size; } - inline Type type() { + inline Type type() const { return _alloc_type; } - inline const char* type_string() { + inline const char* type_string() const { return alloc_type_to_string(_alloc_type); } - inline size_t min_size() { + inline size_t min_size() const { assert (is_lab_alloc(), "Only access for LAB allocs"); return _min_size; } - inline size_t actual_size() { + inline size_t actual_size() const { assert (_actual_size_set, "Should be set"); return _actual_size; } @@ -127,7 +144,15 @@ class ShenandoahAllocRequest : StackObj { _actual_size = v; } - inline bool is_mutator_alloc() { + inline size_t waste() const { + return _waste; + } + + inline void set_waste(size_t v) { + _waste = v; + } + + inline bool is_mutator_alloc() const { switch (_alloc_type) { case _alloc_tlab: case _alloc_shared: @@ -142,7 +167,7 @@ class ShenandoahAllocRequest : StackObj { } } - inline bool is_gc_alloc() { + inline bool is_gc_alloc() const { switch (_alloc_type) { case _alloc_tlab: case _alloc_shared: @@ -157,7 +182,7 @@ class ShenandoahAllocRequest : StackObj { } } - inline bool is_lab_alloc() { + inline bool is_lab_alloc() const { switch (_alloc_type) { case _alloc_tlab: case _alloc_gclab: @@ -172,11 +197,11 @@ class ShenandoahAllocRequest : StackObj { } } - bool is_old() { + bool is_old() const { return _affiliation == OLD_GENERATION; } - bool is_young() { + bool is_young() const { return _affiliation == YOUNG_GENERATION; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index 0f0b2679879ce..8d8c144975506 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -246,13 +246,65 @@ HeapWord* ShenandoahFreeSet::allocate_single(ShenandoahAllocRequest& req, bool& return nullptr; } +// This work method takes an argument corresponding to the number of bytes +// free in a region, and returns the largest amount in heapwords that can be allocated +// such that both of the following conditions are satisfied: +// +// 1. it is a multiple of card size +// 2. any remaining shard may be filled with a filler object +// +// The idea is that the allocation starts and ends at card boundaries. Because +// a region ('s end) is card-aligned, the remainder shard that must be filled is +// at the start of the free space. +// +// This is merely a helper method to use for the purpose of such a calculation. +size_t get_usable_free_words(size_t free_bytes) { + // e.g. card_size is 512, card_shift is 9, min_fill_size() is 8 + // free is 514 + // usable_free is 512, which is decreased to 0 + size_t usable_free = (free_bytes / CardTable::card_size()) << CardTable::card_shift(); + assert(usable_free <= free_bytes, "Sanity check"); + if ((free_bytes != usable_free) && (free_bytes - usable_free < ShenandoahHeap::min_fill_size() * HeapWordSize)) { + // After aligning to card multiples, the remainder would be smaller than + // the minimum filler object, so we'll need to take away another card's + // worth to construct a filler object. + if (usable_free >= CardTable::card_size()) { + usable_free -= CardTable::card_size(); + } else { + assert(usable_free == 0, "usable_free is a multiple of card_size and card_size > min_fill_size"); + } + } + + return usable_free / HeapWordSize; +} + +// Given a size argument, which is a multiple of card size, a request struct +// for a PLAB, and an old region, return a pointer to the allocated space for +// a PLAB which is card-aligned and where any remaining shard in the region +// has been suitably filled by a filler object. +// It is assumed (and assertion-checked) that such an allocation is always possible. +HeapWord* ShenandoahFreeSet::allocate_aligned_plab(size_t size, ShenandoahAllocRequest& req, ShenandoahHeapRegion* r) { + assert(_heap->mode()->is_generational(), "PLABs are only for generational mode"); + assert(r->is_old(), "All PLABs reside in old-gen"); + assert(!req.is_mutator_alloc(), "PLABs should not be allocated by mutators."); + assert(size % CardTable::card_size_in_words() == 0, "size must be multiple of card table size, was " SIZE_FORMAT, size); + + HeapWord* result = r->allocate_aligned(size, req, CardTable::card_size()); + assert(result != nullptr, "Allocation cannot fail"); + assert(r->top() <= r->end(), "Allocation cannot span end of region"); + assert(req.actual_size() == size, "Should not have needed to adjust size for PLAB."); + assert(((uintptr_t) result) % CardTable::card_size_in_words() == 0, "PLAB start must align with card boundary"); + + return result; +} + HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, ShenandoahAllocRequest& req, bool& in_new_region) { assert (!has_no_alloc_capacity(r), "Performance: should avoid full regions on this path: " SIZE_FORMAT, r->index()); - if (_heap->is_concurrent_weak_root_in_progress() && - r->is_trash()) { + if (_heap->is_concurrent_weak_root_in_progress() && r->is_trash()) { return nullptr; } + try_recycle_trashed(r); if (!r->is_affiliated()) { ShenandoahMarkingContext* const ctx = _heap->complete_marking_context(); @@ -278,7 +330,6 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah in_new_region = r->is_empty(); HeapWord* result = nullptr; - size_t size = req.size(); if (in_new_region) { log_debug(gc, free)("Using new region (" SIZE_FORMAT ") for %s (" PTR_FORMAT ").", @@ -288,112 +339,47 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah // req.size() is in words, r->free() is in bytes. if (ShenandoahElasticTLAB && req.is_lab_alloc()) { if (req.type() == ShenandoahAllocRequest::_alloc_plab) { - assert(_heap->mode()->is_generational(), "PLABs are only for generational mode"); // Need to assure that plabs are aligned on multiple of card region. - size_t free = r->free(); - // e.g. card_size is 512, card_shift is 9, min_fill_size() is 8 - // free is 514 - // usable_free is 512, which is decreased to 0 - size_t usable_free = (free / CardTable::card_size()) << CardTable::card_shift(); - if ((free != usable_free) && (free - usable_free < ShenandoahHeap::min_fill_size() * HeapWordSize)) { - // We'll have to add another card's memory to the padding - if (usable_free >= CardTable::card_size()) { - usable_free -= CardTable::card_size(); - } else { - assert(usable_free == 0, "usable_free is a multiple of card_size and card_size > min_fill_size"); - } - } - free /= HeapWordSize; - usable_free /= HeapWordSize; - size_t remnant = size % CardTable::card_size_in_words(); - if (remnant > 0) { - // Since we have Elastic TLABs, align size up. This is consistent with aligning min_size up. - size = size - remnant + CardTable::card_size_in_words(); - } - if (size > usable_free) { - size = usable_free; - assert(size % CardTable::card_size_in_words() == 0, "usable_free is a multiple of card table size"); + // Since we have Elastic TLABs, align sizes up. They may be decreased to fit in the usable + // memory remaining in the region (which will also be aligned to cards). + size_t adjusted_size = align_up(req.size(), CardTable::card_size_in_words()); + size_t adjusted_min_size = align_up(req.min_size(), CardTable::card_size_in_words()); + size_t usable_free = get_usable_free_words(r->free()); + + if (adjusted_size > usable_free) { + adjusted_size = usable_free; } - size_t adjusted_min_size = req.min_size(); - remnant = adjusted_min_size % CardTable::card_size_in_words(); - if (remnant > 0) { - // Round up adjusted_min_size to a multiple of alignment size - adjusted_min_size = adjusted_min_size - remnant + CardTable::card_size_in_words(); - } - if (size >= adjusted_min_size) { - result = r->allocate_aligned(size, req, CardTable::card_size()); - assert(result != nullptr, "Allocation cannot fail"); - size = req.actual_size(); - assert(r->top() <= r->end(), "Allocation cannot span end of region"); - // actual_size() will be set to size below. - assert((result == nullptr) || (size % CardTable::card_size_in_words() == 0), - "PLAB size must be multiple of card size"); - assert((result == nullptr) || (((uintptr_t) result) % CardTable::card_size_in_words() == 0), - "PLAB start must align with card boundary"); - if (free > usable_free) { - // Account for the alignment padding - size_t padding = (free - usable_free) * HeapWordSize; - increase_used(padding); - assert(r->is_old(), "All PLABs reside in old-gen"); - _heap->old_generation()->increase_used(padding); - // For verification consistency, we need to report this padding to _heap - _heap->increase_used(padding); - } + if (adjusted_size >= adjusted_min_size) { + result = allocate_aligned_plab(adjusted_size, req, r); } // Otherwise, leave result == nullptr because the adjusted size is smaller than min size. } else { // This is a GCLAB or a TLAB allocation + size_t adjusted_size = req.size(); size_t free = align_down(r->free() >> LogHeapWordSize, MinObjAlignment); - if (size > free) { - size = free; + if (adjusted_size > free) { + adjusted_size = free; } - if (size >= req.min_size()) { - result = r->allocate(size, req); - if (result != nullptr) { - // Record actual allocation size - req.set_actual_size(size); - } - assert (result != nullptr, "Allocation must succeed: free " SIZE_FORMAT ", actual " SIZE_FORMAT, free, size); + if (adjusted_size >= req.min_size()) { + result = r->allocate(adjusted_size, req); + assert (result != nullptr, "Allocation must succeed: free " SIZE_FORMAT ", actual " SIZE_FORMAT, free, adjusted_size); + req.set_actual_size(adjusted_size); } else { log_trace(gc, free)("Failed to shrink TLAB or GCLAB request (" SIZE_FORMAT ") in region " SIZE_FORMAT " to " SIZE_FORMAT - " because min_size() is " SIZE_FORMAT, req.size(), r->index(), size, req.min_size()); + " because min_size() is " SIZE_FORMAT, req.size(), r->index(), adjusted_size, req.min_size()); } } } else if (req.is_lab_alloc() && req.type() == ShenandoahAllocRequest::_alloc_plab) { - assert(_heap->mode()->is_generational(), "PLABs are only for generational mode"); + // inelastic PLAB - size_t free = r->free(); - size_t usable_free = (free / CardTable::card_size()) << CardTable::card_shift(); - free /= HeapWordSize; - usable_free /= HeapWordSize; - if ((free != usable_free) && (free - usable_free < ShenandoahHeap::min_fill_size() * HeapWordSize)) { - // We'll have to add another card's memory to the padding - if (usable_free > CardTable::card_size_in_words()) { - usable_free -= CardTable::card_size_in_words(); - } else { - assert(usable_free == 0, "usable_free is a multiple of card_size and card_size > min_fill_size"); - } - } - assert(size % CardTable::card_size_in_words() == 0, "PLAB size must be multiple of remembered set card size"); + size_t size = req.size(); + size_t usable_free = get_usable_free_words(r->free()); if (size <= usable_free) { - result = r->allocate_aligned(size, req, CardTable::card_size()); - size = req.actual_size(); - assert(result != nullptr, "Allocation cannot fail"); - assert(r->top() <= r->end(), "Allocation cannot span end of region"); - assert(req.actual_size() % CardTable::card_size_in_words() == 0, "PLAB start must align with card boundary"); - assert(((uintptr_t) result) % CardTable::card_size_in_words() == 0, "PLAB start must align with card boundary"); - if (free > usable_free) { - // Account for the alignment padding - size_t padding = (free - usable_free) * HeapWordSize; - increase_used(padding); - assert(r->is_old(), "All PLABs reside in old-gen"); - _heap->old_generation()->increase_used(padding); - // For verification consistency, we need to report this padding to _heap - _heap->increase_used(padding); - } + result = allocate_aligned_plab(size, req, r); } } else { + size_t size = req.size(); result = r->allocate(size, req); if (result != nullptr) { // Record actual allocation size @@ -405,9 +391,7 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah if (result != nullptr) { // Allocation successful, bump stats: if (req.is_mutator_alloc()) { - assert(req.is_young(), "Mutator allocations always come from young generation."); - generation->increase_used(size * HeapWordSize); - increase_used(size * HeapWordSize); + increase_used(req.actual_size() * HeapWordSize); } else { assert(req.is_gc_alloc(), "Should be gc_alloc since req wasn't mutator alloc"); @@ -419,7 +403,6 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah // PLABs parsable while still allowing the PLAB to serve future allocation requests that arise during the // next evacuation pass. r->set_update_watermark(r->top()); - generation->increase_used(size * HeapWordSize); if (r->is_old()) { assert(req.type() != ShenandoahAllocRequest::_alloc_gclab, "old-gen allocations use PLAB or shared allocation"); // for plabs, we'll sort the difference between evac and promotion usage when we retire the plab @@ -441,8 +424,8 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah size_t waste = r->free(); if (waste > 0) { increase_used(waste); - generation->increase_allocated(waste); - _heap->notify_mutator_alloc_words(waste >> LogHeapWordSize, true); + // This one request could cause several regions to be "retired", so we must accumulate the waste + req.set_waste((waste >> LogHeapWordSize) + req.waste()); } } @@ -584,25 +567,6 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req) { } size_t total_humongous_size = ShenandoahHeapRegion::region_size_bytes() * num; increase_used(total_humongous_size); - if (_heap->mode()->is_generational()) { - size_t humongous_waste = total_humongous_size - words_size * HeapWordSize; - _heap->global_generation()->increase_used(words_size * HeapWordSize); - _heap->global_generation()->increase_humongous_waste(humongous_waste); - if (req.is_young()) { - _heap->young_generation()->increase_used(words_size * HeapWordSize); - _heap->young_generation()->increase_humongous_waste(humongous_waste); - } else if (req.is_old()) { - _heap->old_generation()->increase_used(words_size * HeapWordSize); - _heap->old_generation()->increase_humongous_waste(humongous_waste); - } - } - - if (remainder != 0) { - // Record this remainder as allocation waste - size_t waste = ShenandoahHeapRegion::region_size_words() - remainder; - _heap->notify_mutator_alloc_words(waste, true); - generation->increase_allocated(waste * HeapWordSize); - } // Allocated at left/rightmost? Move the bounds appropriately. if (beg == _mutator_leftmost || end == _mutator_rightmost) { @@ -611,6 +575,9 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req) { assert_bounds(); req.set_actual_size(words_size); + if (remainder != 0) { + req.set_waste(ShenandoahHeapRegion::region_size_words() - remainder); + } return _heap->get_region(beg)->bottom(); } @@ -633,7 +600,6 @@ bool ShenandoahFreeSet::has_no_alloc_capacity(ShenandoahHeapRegion *r) { void ShenandoahFreeSet::try_recycle_trashed(ShenandoahHeapRegion *r) { if (r->is_trash()) { - _heap->decrease_used(r->used()); r->recycle(); } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp index 188163729ec77..a20b1f5873533 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp @@ -50,6 +50,7 @@ class ShenandoahFreeSet : public CHeapObj { bool is_collector_free(size_t idx) const; HeapWord* try_allocate_in(ShenandoahHeapRegion* region, ShenandoahAllocRequest& req, bool& in_new_region); + HeapWord* allocate_aligned_plab(size_t size, ShenandoahAllocRequest& req, ShenandoahHeapRegion* r); HeapWord* allocate_with_affiliation(ShenandoahAffiliation affiliation, ShenandoahAllocRequest& req, bool& in_new_region); HeapWord* allocate_with_old_affiliation(ShenandoahAllocRequest& req, bool& in_new_region); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index 5180ecc47579f..f309dd5375da4 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -1279,14 +1279,12 @@ static void account_for_region(ShenandoahHeapRegion* r, size_t ®ion_count, si class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure { private: ShenandoahHeap* const _heap; - size_t _live; bool _is_generational; size_t _young_regions, _young_usage, _young_humongous_waste; size_t _old_regions, _old_usage, _old_humongous_waste; public: ShenandoahPostCompactClosure() : _heap(ShenandoahHeap::heap()), - _live(0), _is_generational(_heap->mode()->is_generational()), _young_regions(0), _young_usage(0), @@ -1330,29 +1328,33 @@ class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure { if (r->is_trash()) { live = 0; r->recycle(); - } else if (_is_generational) { + } else { if (r->is_old()) { account_for_region(r, _old_regions, _old_usage, _old_humongous_waste); } else if (r->is_young()) { account_for_region(r, _young_regions, _young_usage, _young_humongous_waste); - } else { - // TODO: Assert here? } } r->set_live_data(live); r->reset_alloc_metadata(); - _live += live; - } - - size_t get_live() { - return _live; } void update_generation_usage() { - assert(_is_generational, "Only update generation usage if generational"); - _heap->old_generation()->establish_usage(_old_regions, _old_usage, _old_humongous_waste); - _heap->young_generation()->establish_usage(_young_regions, _young_usage, _young_humongous_waste); + if (_is_generational) { + _heap->old_generation()->establish_usage(_old_regions, _old_usage, _old_humongous_waste); + _heap->young_generation()->establish_usage(_young_regions, _young_usage, _young_humongous_waste); + } else { + assert(_old_regions == 0, "Old regions only expected in generational mode"); + assert(_old_usage == 0, "Old usage only expected in generational mode"); + assert(_old_humongous_waste == 0, "Old humongous waste only expected in generational mode"); + } + + // In generational mode, global usage should be the sum of young and old. This is also true + // for non-generational modes except that there are no old regions. + _heap->global_generation()->establish_usage(_old_regions + _young_regions, + _old_usage + _young_usage, + _old_humongous_waste + _young_humongous_waste); } }; @@ -1361,7 +1363,7 @@ void ShenandoahFullGC::compact_humongous_objects() { // // This code is serial, because doing the in-slice parallel sliding is tricky. In most cases, // humongous regions are already compacted, and do not require further moves, which alleviates - // sliding costs. We may consider doing this in parallel in future. + // sliding costs. We may consider doing this in parallel in the future. ShenandoahHeap* heap = ShenandoahHeap::heap(); @@ -1493,17 +1495,13 @@ void ShenandoahFullGC::phase5_epilog() { ShenandoahGCPhase phase(ShenandoahPhaseTimings::full_gc_copy_objects_rebuild); ShenandoahPostCompactClosure post_compact; heap->heap_region_iterate(&post_compact); - heap->set_used(post_compact.get_live()); - if (heap->mode()->is_generational()) { - post_compact.update_generation_usage(); - log_info(gc)("FullGC done: GLOBAL usage: " SIZE_FORMAT "%s, young usage: " SIZE_FORMAT "%s, old usage: " SIZE_FORMAT "%s", - byte_size_in_proper_unit(post_compact.get_live()), proper_unit_for_byte_size(post_compact.get_live()), - byte_size_in_proper_unit(heap->young_generation()->used()), proper_unit_for_byte_size(heap->young_generation()->used()), - byte_size_in_proper_unit(heap->old_generation()->used()), proper_unit_for_byte_size(heap->old_generation()->used())); - } + post_compact.update_generation_usage(); + log_info(gc)("FullGC done: global usage: " SIZE_FORMAT "%s, young usage: " SIZE_FORMAT "%s, old usage: " SIZE_FORMAT "%s", + byte_size_in_proper_unit(heap->global_generation()->used()), proper_unit_for_byte_size(heap->global_generation()->used()), + byte_size_in_proper_unit(heap->young_generation()->used()), proper_unit_for_byte_size(heap->young_generation()->used()), + byte_size_in_proper_unit(heap->old_generation()->used()), proper_unit_for_byte_size(heap->old_generation()->used())); heap->collection_set()->clear(); heap->free_set()->rebuild(); } heap->clear_cancelled_gc(true /* clear oom handler */); } - diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index bb71625d94187..bbdb6977822eb 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -137,7 +137,7 @@ ShenandoahHeuristics* ShenandoahGeneration::initialize_heuristics(ShenandoahMode } size_t ShenandoahGeneration::bytes_allocated_since_gc_start() { - return Atomic::load(&_bytes_allocated_since_gc_start);; + return Atomic::load(&_bytes_allocated_since_gc_start); } void ShenandoahGeneration::reset_bytes_allocated_since_gc_start() { @@ -405,7 +405,7 @@ void ShenandoahGeneration::compute_evacuation_budgets(ShenandoahHeap* heap, bool collection_set->establish_preselected(preselected_regions); } -// Having chosen the collection set, adjust the budgets for generatioal mode based on its composition. Note +// Having chosen the collection set, adjust the budgets for generational mode based on its composition. Note // that young_generation->available() now knows about recently discovered immediate garbage. void ShenandoahGeneration::adjust_evacuation_budgets(ShenandoahHeap* heap, ShenandoahCollectionSet* collection_set, @@ -956,20 +956,12 @@ size_t ShenandoahGeneration::decrement_affiliated_region_count() { } void ShenandoahGeneration::establish_usage(size_t num_regions, size_t num_bytes, size_t humongous_waste) { - assert(ShenandoahHeap::heap()->mode()->is_generational(), "Only generational mode accounts for generational usage"); assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at a safepoint"); _affiliated_region_count = num_regions; _used = num_bytes; _humongous_waste = humongous_waste; } -void ShenandoahGeneration::clear_used() { - assert(ShenandoahHeap::heap()->mode()->is_generational(), "Only generational mode accounts for generational usage"); - assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "must be at a safepoint"); - // Do this atomically to assure visibility to other threads, even though these other threads may be idle "right now".. - Atomic::store(&_used, (size_t)0); -} - void ShenandoahGeneration::increase_used(size_t bytes) { Atomic::add(&_used, bytes); } @@ -981,18 +973,16 @@ void ShenandoahGeneration::decrease_used(size_t bytes) { void ShenandoahGeneration::increase_humongous_waste(size_t bytes) { if (bytes > 0) { - shenandoah_assert_heaplocked_or_fullgc_safepoint(); - _humongous_waste += bytes; + Atomic::add(&_humongous_waste, bytes); } } void ShenandoahGeneration::decrease_humongous_waste(size_t bytes) { if (bytes > 0) { - shenandoah_assert_heaplocked_or_fullgc_safepoint(); assert(_humongous_waste >= bytes, "Waste cannot be negative"); assert(ShenandoahHeap::heap()->is_full_gc_in_progress() || (_humongous_waste >= bytes), "Waste (" SIZE_FORMAT ") cannot be negative (after subtracting " SIZE_FORMAT ")", _humongous_waste, bytes); - _humongous_waste -= bytes; + Atomic::sub(&_humongous_waste, bytes); } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index 9e9385e5fa147..9a92c966d9351 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -38,6 +38,7 @@ class ShenandoahHeap; class ShenandoahMode; class ShenandoahGeneration : public CHeapObj { + friend class VMStructs; private: ShenandoahGenerationType const _type; @@ -200,7 +201,6 @@ class ShenandoahGeneration : public CHeapObj { void establish_usage(size_t num_regions, size_t num_bytes, size_t humongous_waste); - void clear_used(); void increase_used(size_t bytes); void decrease_used(size_t bytes); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp index 3259c91aa5f62..6c8acd1ec5fe2 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp @@ -53,10 +53,6 @@ size_t ShenandoahGlobalGeneration::soft_max_capacity() const { return ShenandoahHeap::heap()->soft_max_capacity(); } -size_t ShenandoahGlobalGeneration::used() const { - return ShenandoahHeap::heap()->used(); -} - size_t ShenandoahGlobalGeneration::available() const { return ShenandoahHeap::heap()->free_set()->available(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp index 0d94ce56ad38d..a745b8ec5b27e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.hpp @@ -42,7 +42,6 @@ class ShenandoahGlobalGeneration : public ShenandoahGeneration { virtual size_t soft_max_capacity() const override; virtual size_t used_regions() const override; virtual size_t used_regions_size() const override; - virtual size_t used() const override; virtual size_t available() const override; virtual void set_concurrent_mark_in_progress(bool in_progress) override; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index e146b486416ee..b7ed4dda4d558 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -36,6 +36,7 @@ #include "gc/shared/plab.hpp" #include "gc/shared/tlab_globals.hpp" +#include "gc/shenandoah/shenandoahAllocRequest.hpp" #include "gc/shenandoah/shenandoahBarrierSet.hpp" #include "gc/shenandoah/shenandoahCardTable.hpp" #include "gc/shenandoah/shenandoahClosures.inline.hpp" @@ -77,6 +78,7 @@ #include "gc/shenandoah/mode/shenandoahIUMode.hpp" #include "gc/shenandoah/mode/shenandoahPassiveMode.hpp" #include "gc/shenandoah/mode/shenandoahSATBMode.hpp" +#include "utilities/globalDefinitions.hpp" #if INCLUDE_JFR #include "gc/shenandoah/shenandoahJfrSupport.hpp" @@ -539,7 +541,6 @@ ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) : _gc_generation(nullptr), _prepare_for_old_mark(false), _initial_size(0), - _used(0), _committed(0), _max_workers(MAX3(ConcGCThreads, ParallelGCThreads, 1U)), _workers(nullptr), @@ -696,7 +697,7 @@ bool ShenandoahHeap::is_gc_generation_young() const { } size_t ShenandoahHeap::used() const { - return Atomic::load(&_used); + return global_generation()->used(); } size_t ShenandoahHeap::committed() const { @@ -713,29 +714,84 @@ void ShenandoahHeap::decrease_committed(size_t bytes) { _committed -= bytes; } -void ShenandoahHeap::increase_used(size_t bytes) { - Atomic::add(&_used, bytes, memory_order_relaxed); +// For tracking usage based on allocations, it should be the case that: +// * The sum of regions::used == heap::used +// * The sum of a generation's regions::used == generation::used +// * The sum of a generation's humongous regions::free == generation::humongous_waste +// These invariants are checked by the verifier on GC safepoints. +// +// Additional notes: +// * When a mutator's allocation request causes a region to be retired, the +// free memory left in that region is considered waste. It does not contribute +// to the usage, but it _does_ contribute to allocation rate. +// * The bottom of a PLAB must be aligned on card size. In some cases this will +// require padding in front of the PLAB (a filler object). Because this padding +// is included in the region's used memory we include the padding in the usage +// accounting as waste. +// * Mutator allocations are used to compute an allocation rate. They are also +// sent to the Pacer for those purposes. +// * There are three sources of waste: +// 1. The padding used to align a PLAB on card size +// 2. Region's free is less than minimum TLAB size and is retired +// 3. The unused portion of memory in the last region of a humongous object +void ShenandoahHeap::increase_used(const ShenandoahAllocRequest& req) { + size_t actual_bytes = req.actual_size() * HeapWordSize; + size_t wasted_bytes = req.waste() * HeapWordSize; + ShenandoahGeneration* generation = generation_for(req.affiliation()); + + if (req.is_gc_alloc()) { + assert(wasted_bytes == 0 || req.type() == ShenandoahAllocRequest::_alloc_plab, "Only PLABs have waste"); + increase_used(generation, actual_bytes + wasted_bytes); + } else { + assert(req.is_mutator_alloc(), "Expected mutator alloc here"); + // padding and actual size both count towards allocation counter + generation->increase_allocated(actual_bytes + wasted_bytes); + + // only actual size counts toward usage for mutator allocations + increase_used(generation, actual_bytes); + + // notify pacer of both actual size and waste + notify_mutator_alloc_words(req.actual_size(), req.waste()); + + if (wasted_bytes > 0 && req.actual_size() > ShenandoahHeapRegion::humongous_threshold_words()) { + increase_humongous_waste(generation,wasted_bytes); + } + } } -void ShenandoahHeap::set_used(size_t bytes) { - Atomic::store(&_used, bytes); +void ShenandoahHeap::increase_humongous_waste(ShenandoahGeneration* generation, size_t bytes) { + generation->increase_humongous_waste(bytes); + if (!generation->is_global()) { + global_generation()->increase_humongous_waste(bytes); + } } -void ShenandoahHeap::decrease_used(size_t bytes) { - assert(used() >= bytes, "never decrease heap size by more than we've left"); - Atomic::sub(&_used, bytes, memory_order_relaxed); +void ShenandoahHeap::decrease_humongous_waste(ShenandoahGeneration* generation, size_t bytes) { + generation->decrease_humongous_waste(bytes); + if (!generation->is_global()) { + global_generation()->decrease_humongous_waste(bytes); + } } -void ShenandoahHeap::notify_mutator_alloc_words(size_t words, bool waste) { - size_t bytes = words * HeapWordSize; - if (!waste) { - increase_used(bytes); +void ShenandoahHeap::increase_used(ShenandoahGeneration* generation, size_t bytes) { + generation->increase_used(bytes); + if (!generation->is_global()) { + global_generation()->increase_used(bytes); } +} +void ShenandoahHeap::decrease_used(ShenandoahGeneration* generation, size_t bytes) { + generation->decrease_used(bytes); + if (!generation->is_global()) { + global_generation()->decrease_used(bytes); + } +} + +void ShenandoahHeap::notify_mutator_alloc_words(size_t words, size_t waste) { if (ShenandoahPacing) { control_thread()->pacing_notify_alloc(words); - if (waste) { - pacer()->claim_for_alloc(words, true); + if (waste > 0) { + pacer()->claim_for_alloc(waste, true); } } } @@ -1196,28 +1252,29 @@ HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req, bool is_p regulator_thread()->notify_heap_changed(); } + if (result == nullptr) { + req.set_actual_size(0); + } + + // This is called regardless of the outcome of the allocation to account + // for any waste created by retiring regions with this request. + increase_used(req); + if (result != nullptr) { - ShenandoahGeneration* alloc_generation = generation_for(req.affiliation()); size_t requested = req.size(); size_t actual = req.actual_size(); - size_t actual_bytes = actual * HeapWordSize; assert (req.is_lab_alloc() || (requested == actual), "Only LAB allocations are elastic: %s, requested = " SIZE_FORMAT ", actual = " SIZE_FORMAT, ShenandoahAllocRequest::alloc_type_to_string(req.type()), requested, actual); if (req.is_mutator_alloc()) { - notify_mutator_alloc_words(actual, false); - alloc_generation->increase_allocated(actual_bytes); - // If we requested more than we were granted, give the rest back to pacer. // This only matters if we are in the same pacing epoch: do not try to unpace // over the budget for the other phase. if (ShenandoahPacing && (pacer_epoch > 0) && (requested > actual)) { pacer()->unpace_for_alloc(pacer_epoch, requested - actual); } - } else { - increase_used(actual_bytes); } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index 4e0bc45cb7f6b..fecc41fb39008 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -217,14 +217,16 @@ class ShenandoahHeap : public CollectedHeap { size_t _minimum_size; volatile size_t _soft_max_size; shenandoah_padding(0); - volatile size_t _used; volatile size_t _committed; shenandoah_padding(1); + void increase_used(const ShenandoahAllocRequest& req); + public: - void increase_used(size_t bytes); - void decrease_used(size_t bytes); - void set_used(size_t bytes); + void increase_used(ShenandoahGeneration* generation, size_t bytes); + void decrease_used(ShenandoahGeneration* generation, size_t bytes); + void increase_humongous_waste(ShenandoahGeneration* generation, size_t bytes); + void decrease_humongous_waste(ShenandoahGeneration* generation, size_t bytes); void increase_committed(size_t bytes); void decrease_committed(size_t bytes); @@ -703,7 +705,7 @@ class ShenandoahHeap : public CollectedHeap { size_t size, Metaspace::MetadataType mdtype) override; - void notify_mutator_alloc_words(size_t words, bool waste); + void notify_mutator_alloc_words(size_t words, size_t waste); HeapWord* allocate_new_tlab(size_t min_size, size_t requested_size, size_t* actual_size) override; size_t tlab_capacity(Thread *thr) const override; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index a3afa6d2b83e1..299c2b1c593f2 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -287,9 +287,7 @@ void ShenandoahHeapRegion::make_trash() { { // Reclaiming humongous regions and reclaim humongous waste. When this region is eventually recycled, we'll reclaim // its used memory. At recycle time, we no longer recognize this as a humongous region. - if (ShenandoahHeap::heap()->mode()->is_generational()) { - decrement_humongous_waste(); - } + decrement_humongous_waste(); } case _cset: // Reclaiming cset regions @@ -658,12 +656,10 @@ ShenandoahHeapRegion* ShenandoahHeapRegion::humongous_start_region() const { } void ShenandoahHeapRegion::recycle() { - ShenandoahHeap* heap = ShenandoahHeap::heap(); shenandoah_assert_heaplocked(); - - if (ShenandoahHeap::heap()->mode()->is_generational()) { - heap->generation_for(affiliation())->decrease_used(used()); - } + ShenandoahHeap* heap = ShenandoahHeap::heap(); + ShenandoahGeneration* generation = heap->generation_for(affiliation()); + heap->decrease_used(generation, used()); set_top(bottom()); clear_live_data(); @@ -1093,7 +1089,8 @@ void ShenandoahHeapRegion::decrement_humongous_waste() const { assert(is_humongous(), "Should only use this for humongous regions"); size_t waste_bytes = free(); if (waste_bytes > 0) { - ShenandoahHeap::heap()->generation_for(affiliation())->decrease_humongous_waste(waste_bytes); - ShenandoahHeap::heap()->global_generation()->decrease_humongous_waste(waste_bytes); + ShenandoahHeap* heap = ShenandoahHeap::heap(); + ShenandoahGeneration* generation = heap->generation_for(affiliation()); + heap->decrease_humongous_waste(generation, waste_bytes); } } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp index 88f84da09e351..4ef7f36ab1df9 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp @@ -93,6 +93,7 @@ HeapWord* ShenandoahHeapRegion::allocate_aligned(size_t size, ShenandoahAllocReq assert(new_top <= end(), "PLAB cannot span end of heap region"); set_top(new_top); req.set_actual_size(size); + req.set_waste(pad_words); assert(is_object_aligned(new_top), "new top breaks alignment: " PTR_FORMAT, p2i(new_top)); assert(is_aligned(aligned_obj, alignment_in_bytes), "obj is not aligned: " PTR_FORMAT, p2i(aligned_obj)); return aligned_obj; diff --git a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp index ab2e97029c06a..bec9b9cb53992 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp @@ -274,7 +274,8 @@ " 4 = previous level, plus all marked objects") \ \ product(bool, ShenandoahElasticTLAB, true, DIAGNOSTIC, \ - "Use Elastic TLABs with Shenandoah") \ + "Use Elastic TLABs with Shenandoah. This allows Shenandoah to " \ + "decrease the size of a TLAB to fit in a region's remaining space") \ \ product(uintx, ShenandoahEvacReserve, 5, EXPERIMENTAL, \ "How much of (young-generation) heap to reserve for " \ diff --git a/src/hotspot/share/gc/shenandoah/vmStructs_shenandoah.hpp b/src/hotspot/share/gc/shenandoah/vmStructs_shenandoah.hpp index 9bc3af5ba9e03..c0763ba76eee6 100644 --- a/src/hotspot/share/gc/shenandoah/vmStructs_shenandoah.hpp +++ b/src/hotspot/share/gc/shenandoah/vmStructs_shenandoah.hpp @@ -25,6 +25,7 @@ #define SHARE_GC_SHENANDOAH_VMSTRUCTS_SHENANDOAH_HPP #include "gc/shenandoah/shenandoahHeap.hpp" +#include "gc/shenandoah/shenandoahGeneration.hpp" #include "gc/shenandoah/shenandoahHeapRegion.hpp" #include "gc/shenandoah/shenandoahMonitoringSupport.hpp" @@ -32,8 +33,9 @@ nonstatic_field(ShenandoahHeap, _num_regions, size_t) \ nonstatic_field(ShenandoahHeap, _regions, ShenandoahHeapRegion**) \ nonstatic_field(ShenandoahHeap, _log_min_obj_alignment_in_bytes, int) \ - volatile_nonstatic_field(ShenandoahHeap, _used, size_t) \ + nonstatic_field(ShenandoahHeap, _global_generation, ShenandoahGeneration*) \ volatile_nonstatic_field(ShenandoahHeap, _committed, size_t) \ + volatile_nonstatic_field(ShenandoahGeneration, _used, size_t) \ static_field(ShenandoahHeapRegion, RegionSizeBytes, size_t) \ static_field(ShenandoahHeapRegion, RegionSizeBytesShift, size_t) \ nonstatic_field(ShenandoahHeapRegion, _state, ShenandoahHeapRegion::RegionState) \ @@ -62,5 +64,7 @@ declare_toplevel_type(ShenandoahHeap*) \ declare_toplevel_type(ShenandoahHeapRegion*) \ declare_toplevel_type(ShenandoahHeapRegion::RegionState) \ + declare_toplevel_type(ShenandoahGeneration) \ + declare_toplevel_type(ShenandoahGeneration*) \ #endif // SHARE_GC_SHENANDOAH_VMSTRUCTS_SHENANDOAH_HPP diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/shenandoah/ShenandoahGeneration.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/shenandoah/ShenandoahGeneration.java new file mode 100644 index 0000000000000..bcd59523ae088 --- /dev/null +++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/shenandoah/ShenandoahGeneration.java @@ -0,0 +1,59 @@ +/* + * * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +package sun.jvm.hotspot.gc.shenandoah; + +import sun.jvm.hotspot.utilities.Observable; +import sun.jvm.hotspot.utilities.Observer; + +import sun.jvm.hotspot.debugger.Address; +import sun.jvm.hotspot.runtime.VM; +import sun.jvm.hotspot.runtime.VMObject; +import sun.jvm.hotspot.types.CIntegerField; +import sun.jvm.hotspot.types.Type; +import sun.jvm.hotspot.types.TypeDataBase; + +public class ShenandoahGeneration extends VMObject { + private static CIntegerField used; + static { + VM.registerVMInitializedObserver(new Observer() { + public void update(Observable o, Object data) { + initialize(VM.getVM().getTypeDataBase()); + } + }); + } + + private static synchronized void initialize(TypeDataBase db) { + Type type = db.lookupType("ShenandoahGeneration"); + used = type.getCIntegerField("_used"); + } + + public ShenandoahGeneration(Address addr) { + super(addr); + } + + public long used() { + return used.getValue(addr); + } +} diff --git a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/shenandoah/ShenandoahHeap.java b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/shenandoah/ShenandoahHeap.java index ca12562ac3e0b..3109fe2210290 100644 --- a/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/shenandoah/ShenandoahHeap.java +++ b/src/jdk.hotspot.agent/share/classes/sun/jvm/hotspot/gc/shenandoah/ShenandoahHeap.java @@ -43,7 +43,7 @@ public class ShenandoahHeap extends CollectedHeap { private static CIntegerField numRegions; - private static CIntegerField used; + private static AddressField globalGeneration; private static CIntegerField committed; private static AddressField regions; private static CIntegerField logMinObjAlignmentInBytes; @@ -60,7 +60,7 @@ public void update(Observable o, Object data) { private static synchronized void initialize(TypeDataBase db) { Type type = db.lookupType("ShenandoahHeap"); numRegions = type.getCIntegerField("_num_regions"); - used = type.getCIntegerField("_used"); + globalGeneration = type.getAddressField("_global_generation"); committed = type.getCIntegerField("_committed"); regions = type.getAddressField("_regions"); logMinObjAlignmentInBytes = type.getCIntegerField("_log_min_obj_alignment_in_bytes"); @@ -89,7 +89,9 @@ public long capacity() { @Override public long used() { - return used.getValue(addr); + Address globalGenerationAddress = globalGeneration.getValue(addr); + ShenandoahGeneration global = VMObjectFactory.newObject(ShenandoahGeneration.class, globalGenerationAddress); + return global.used(); } public long committed() { From b442302c4dbbc7063cb43836a8c833df68c48a7d Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Wed, 26 Apr 2023 16:26:56 +0000 Subject: [PATCH 225/254] Remove redundant initialization code Reviewed-by: shade --- .../share/gc/shenandoah/shenandoahFreeSet.cpp | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index 8d8c144975506..db955d5bd2d7e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -549,18 +549,7 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req) { r->set_affiliation(req.affiliation()); r->set_update_watermark(r->bottom()); - r->set_top(r->bottom()); // Set top to bottom so we can capture TAMS - ctx->capture_top_at_mark_start(r); - r->set_top(r->bottom() + used_words); // Then change top to reflect allocation of humongous object. - assert(ctx->top_at_mark_start(r) == r->bottom(), "Newly established allocation region starts with TAMS equal to bottom"); - assert(ctx->is_bitmap_clear_range(ctx->top_bitmap(r), r->end()), "Bitmap above top_bitmap() must be clear"); - - // Leave top_bitmap alone. The first time a heap region is put into service, top_bitmap should equal end. - // Thereafter, it should represent the upper bound on parts of the bitmap that need to be cleared. - // ctx->clear_bitmap(r); - log_debug(gc, free)("NOT clearing bitmap for Humongous region [" PTR_FORMAT ", " PTR_FORMAT "], top_bitmap: " - PTR_FORMAT " at transition from FREE to %s", - p2i(r->bottom()), p2i(r->end()), p2i(ctx->top_bitmap(r)), req.affiliation_name()); + r->set_top(r->bottom() + used_words); // While individual regions report their true use, all humongous regions are marked used in the free set. _mutator_free_bitmap.clear_bit(r->index()); From 58fa1410a8ad1462cbe7c035b7d64f44a0b62c85 Mon Sep 17 00:00:00 2001 From: Aleksey Shipilev Date: Mon, 1 May 2023 07:53:53 +0000 Subject: [PATCH 226/254] Cleanup NumberSeq additions Reviewed-by: ysr, rkennke --- .../gc/shenandoah/shenandoahNumberSeq.cpp | 88 ++++++++++++------- .../gc/shenandoah/shenandoahNumberSeq.hpp | 5 +- .../shenandoah/shenandoahScanRemembered.hpp | 6 +- .../shenandoahScanRemembered.inline.hpp | 15 ++-- src/hotspot/share/utilities/numberSeq.cpp | 42 --------- src/hotspot/share/utilities/numberSeq.hpp | 11 +-- .../shenandoah/test_shenandoahNumberSeq.cpp | 67 +++++++++----- 7 files changed, 118 insertions(+), 116 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.cpp b/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.cpp index 5f77bcc44c90c..3c7ba8e424300 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.cpp @@ -121,40 +121,68 @@ double HdrSeq::percentile(double level) const { return maximum(); } -// Merge this HdrSeq into hdr2: clear optional and on-by-default -// Note: this method isn't intrinsically MT-safe; callers must take care -// of any mutual exclusion as necessary. -void HdrSeq::merge(HdrSeq& hdr2, bool clear_this) { +void HdrSeq::add(const HdrSeq& other) { + if (other.num() == 0) { + // Other sequence is empty, return + return; + } + for (int mag = 0; mag < MagBuckets; mag++) { - if (_hdr[mag] != nullptr) { - int* that_bucket = hdr2._hdr[mag]; - if (that_bucket == nullptr) { - if (clear_this) { - // the target doesn't have any values, swap in ours. - // Could this cause native memory fragmentation? - hdr2._hdr[mag] = _hdr[mag]; - _hdr[mag] = nullptr; - } else { - // We can't clear this, so we create the entries & add in below - that_bucket = NEW_C_HEAP_ARRAY(int, ValBuckets, mtInternal); - for (int val = 0; val < ValBuckets; val++) { - that_bucket[val] = _hdr[mag][val]; - } - hdr2._hdr[mag] = that_bucket; - } - } else { - // Add in our values into target - for (int val = 0; val < ValBuckets; val++) { - that_bucket[val] += _hdr[mag][val]; - if (clear_this) { - _hdr[mag][val] = 0; - } - } + int* other_bucket = other._hdr[mag]; + if (other_bucket == nullptr) { + // Nothing to do + continue; + } + int* bucket = _hdr[mag]; + if (bucket != nullptr) { + // Add into our bucket + for (int val = 0; val < ValBuckets; val++) { + bucket[val] += other_bucket[val]; } + } else { + // Create our bucket and copy the contents over + bucket = NEW_C_HEAP_ARRAY(int, ValBuckets, mtInternal); + for (int val = 0; val < ValBuckets; val++) { + bucket[val] = other_bucket[val]; + } + _hdr[mag] = bucket; } } - // Merge up the class hierarchy - NumberSeq::merge(hdr2, clear_this); + + // This is a hacky way to only update the fields we want. + // This inlines NumberSeq code without going into AbsSeq and + // dealing with decayed average/variance, which we do not + // know how to compute yet. + _last = other._last; + _maximum = MAX2(_maximum, other._maximum); + _sum += other._sum; + _sum_of_squares += other._sum_of_squares; + _num += other._num; + + // Until JDK-8298902 is fixed, we taint the decaying statistics + _davg = NAN; + _dvariance = NAN; +} + +void HdrSeq::clear() { + // Clear the storage + for (int mag = 0; mag < MagBuckets; mag++) { + int* bucket = _hdr[mag]; + if (bucket != nullptr) { + for (int c = 0; c < ValBuckets; c++) { + bucket[c] = 0; + } + } + } + + // Clear other fields too + _last = 0; + _maximum = 0; + _sum = 0; + _sum_of_squares = 0; + _num = 0; + _davg = 0; + _dvariance = 0; } BinaryMagnitudeSeq::BinaryMagnitudeSeq() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.hpp b/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.hpp index 2249d1ac2fa98..8dc75d0f3acbb 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.hpp @@ -50,10 +50,9 @@ class HdrSeq: public NumberSeq { ~HdrSeq(); virtual void add(double val); + void add(const HdrSeq& other); double percentile(double level) const; - - // Merge this HdrSeq into hdr2, optionally clearing this HdrSeq - void merge(HdrSeq& hdr2, bool clear_this = true); + void clear(); }; // Binary magnitude sequence stores the power-of-two histogram. diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp index 0c719d4abfcde..0c92a5eceeb47 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp @@ -927,14 +927,14 @@ class ShenandoahScanRemembered: public CHeapObj { // Log stats related to card/RS stats for given phase t void log_card_stats(uint nworkers, CardStatLogType t) PRODUCT_RETURN; private: - // Log stats for given worker id related into given cumulative card/RS stats - void log_worker_card_stats(uint worker_id, HdrSeq* cum_stats) PRODUCT_RETURN; + // Log stats for given worker id related into given summary card/RS stats + void log_worker_card_stats(uint worker_id, HdrSeq* sum_stats) PRODUCT_RETURN; // Log given stats inline void log_card_stats(HdrSeq* stats) PRODUCT_RETURN; // Merge the stats from worked_id into the given summary stats, and clear the worker_id's stats. - void merge_worker_card_stats_cumulative(HdrSeq* worker_stats, HdrSeq* cum_stats) PRODUCT_RETURN; + void merge_worker_card_stats_cumulative(HdrSeq* worker_stats, HdrSeq* sum_stats) PRODUCT_RETURN; }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp index 465c6b8fb392b..42fede2943eb6 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.inline.hpp @@ -936,37 +936,38 @@ inline void ShenandoahScanRemembered::log_card_stats(HdrSeq* stat template void ShenandoahScanRemembered::log_card_stats(uint nworkers, CardStatLogType t) { assert(ShenandoahEnableCardStats, "Do not call"); - HdrSeq* cum_stats = card_stats_for_phase(t); + HdrSeq* sum_stats = card_stats_for_phase(t); log_info(gc, remset)("%s", _card_stat_log_type[t]); for (uint i = 0; i < nworkers; i++) { - log_worker_card_stats(i, cum_stats); + log_worker_card_stats(i, sum_stats); } // Every so often, log the cumulative global stats if (++_card_stats_log_counter[t] >= ShenandoahCardStatsLogInterval) { _card_stats_log_counter[t] = 0; log_info(gc, remset)("Cumulative stats"); - log_card_stats(cum_stats); + log_card_stats(sum_stats); } } // Log card stats for given worker_id, & clear them after merging into given cumulative stats template -void ShenandoahScanRemembered::log_worker_card_stats(uint worker_id, HdrSeq* cum_stats) { +void ShenandoahScanRemembered::log_worker_card_stats(uint worker_id, HdrSeq* sum_stats) { assert(ShenandoahEnableCardStats, "Do not call"); HdrSeq* worker_card_stats = card_stats(worker_id); log_info(gc, remset)("Worker %u Card Stats: ", worker_id); log_card_stats(worker_card_stats); // Merge worker stats into the cumulative stats & clear worker stats - merge_worker_card_stats_cumulative(worker_card_stats, cum_stats); + merge_worker_card_stats_cumulative(worker_card_stats, sum_stats); } template void ShenandoahScanRemembered::merge_worker_card_stats_cumulative( - HdrSeq* worker_stats, HdrSeq* cum_stats) { + HdrSeq* worker_stats, HdrSeq* sum_stats) { for (int i = 0; i < MAX_CARD_STAT_TYPE; i++) { - worker_stats[i].merge(cum_stats[i]); + sum_stats[i].add(worker_stats[i]); + worker_stats[i].clear(); } } #endif diff --git a/src/hotspot/share/utilities/numberSeq.cpp b/src/hotspot/share/utilities/numberSeq.cpp index 304e2c3e20e5a..db2f8d92d739a 100644 --- a/src/hotspot/share/utilities/numberSeq.cpp +++ b/src/hotspot/share/utilities/numberSeq.cpp @@ -110,32 +110,6 @@ double AbsSeq::dsd() const { return sqrt(var); } -void AbsSeq::merge(AbsSeq& abs2, bool clear_this) { - - if (num() == 0) return; // nothing to do - - abs2._num += _num; - abs2._sum += _sum; - abs2._sum_of_squares += _sum_of_squares; - - // Decaying stats need a bit more thought - assert(abs2._alpha == _alpha, "Caution: merge incompatible?"); - // Until JDK-8298902 is fixed, we taint the decaying statistics - if (abs2._davg != NAN) { - abs2._davg = NAN; - abs2._dvariance = NAN; - } - - if (clear_this) { - _num = 0; - _sum = 0; - _sum_of_squares = 0; - _davg = 0; - _dvariance = 0; - } -} - - NumberSeq::NumberSeq(double alpha) : AbsSeq(alpha), _last(0.0), _maximum(0.0) { } @@ -163,22 +137,6 @@ void NumberSeq::add(double val) { ++_num; } -void NumberSeq::merge(NumberSeq& nseq2, bool clear_this) { - - if (num() == 0) return; // nothing to do - - nseq2._last = _last; // this is newer than that - nseq2._maximum = MAX2(_maximum, nseq2._maximum); - - AbsSeq::merge(nseq2, clear_this); - - if (clear_this) { - _last = 0; - _maximum = 0; - assert(num() == 0, "Not cleared"); - } -} - TruncatedSeq::TruncatedSeq(int length, double alpha): AbsSeq(alpha), _length(length), _next(0) { diff --git a/src/hotspot/share/utilities/numberSeq.hpp b/src/hotspot/share/utilities/numberSeq.hpp index 6e802aa598168..b57fdf45576ab 100644 --- a/src/hotspot/share/utilities/numberSeq.hpp +++ b/src/hotspot/share/utilities/numberSeq.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2019, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -83,9 +83,6 @@ class AbsSeq: public CHeapObj { // Debugging/Printing virtual void dump(); virtual void dump_on(outputStream* s); - - // Merge this AbsSeq into seq2, optionally clearing this AbsSeq - void merge(AbsSeq& seq2, bool clear_this = true); }; class NumberSeq: public AbsSeq { @@ -105,9 +102,6 @@ class NumberSeq: public AbsSeq { // Debugging/Printing virtual void dump_on(outputStream* s); - - // Merge this NumberSeq into seq2, optionally clearing this NumberSeq - void merge(NumberSeq& seq2, bool clear_this = true); }; class TruncatedSeq: public AbsSeq { @@ -135,9 +129,6 @@ class TruncatedSeq: public AbsSeq { // Debugging/Printing virtual void dump_on(outputStream* s); - - // Merge this AbsSeq into seq2, optionally clearing this AbsSeq - void merge(AbsSeq& seq2, bool clear_this = true); }; #endif // SHARE_UTILITIES_NUMBERSEQ_HPP diff --git a/test/hotspot/gtest/gc/shenandoah/test_shenandoahNumberSeq.cpp b/test/hotspot/gtest/gc/shenandoah/test_shenandoahNumberSeq.cpp index 56c24788a82fa..ce0ddf434acda 100644 --- a/test/hotspot/gtest/gc/shenandoah/test_shenandoahNumberSeq.cpp +++ b/test/hotspot/gtest/gc/shenandoah/test_shenandoahNumberSeq.cpp @@ -109,30 +109,55 @@ TEST_VM_F(BasicShenandoahNumberSeqTest, percentile_test) { EXPECT_NEAR(100, seq1.percentile(100), err); } +TEST_VM_F(BasicShenandoahNumberSeqTest, clear_test) { + HdrSeq test; + test.add(1); + + EXPECT_NE(test.num(), 0); + EXPECT_NE(test.sum(), 0); + EXPECT_NE(test.maximum(), 0); + EXPECT_NE(test.avg(), 0); + EXPECT_EQ(test.sd(), 0); + EXPECT_NE(test.davg(), 0); + EXPECT_EQ(test.dvariance(), 0); + for (int i = 0; i <= 100; i += 10) { + EXPECT_NE(test.percentile(i), 0); + } + + test.clear(); + + EXPECT_EQ(test.num(), 0); + EXPECT_EQ(test.sum(), 0); + EXPECT_EQ(test.maximum(), 0); + EXPECT_EQ(test.avg(), 0); + EXPECT_EQ(test.sd(), 0); + EXPECT_EQ(test.davg(), 0); + EXPECT_EQ(test.dvariance(), 0); + for (int i = 0; i <= 100; i += 10) { + EXPECT_EQ(test.percentile(i), 0); + } +} + TEST_VM_F(ShenandoahNumberSeqMergeTest, merge_test) { EXPECT_EQ(seq1.num(), 80); EXPECT_EQ(seq2.num(), 20); - EXPECT_FALSE(isnan(seq2.davg())); // Exercise the path; not a nan - EXPECT_FALSE(isnan(seq2.dsd())); - EXPECT_FALSE(isnan(seq2.dvariance())); - - std::cout << "Pre-merge: \n"; - print(); - seq1.merge(seq2); // clears seq1, after merging into seq2 - std::cout << "Post-merge: \n"; - print(); - - EXPECT_EQ(seq1.num(), 0); - EXPECT_EQ(seq2.num(), 100); - EXPECT_EQ(seq2.num(), seq3.num()); - EXPECT_TRUE(isnan(seq2.davg())); // until we fix decayed stats - EXPECT_TRUE(isnan(seq2.dvariance())); - - EXPECT_EQ(seq2.maximum(), seq3.maximum()); - EXPECT_EQ(seq2.percentile(0), seq3.percentile(0)); + EXPECT_EQ(seq3.num(), 100); + + HdrSeq merged; + merged.add(seq1); + merged.add(seq2); + + EXPECT_EQ(merged.num(), seq3.num()); + + EXPECT_EQ(merged.maximum(), seq3.maximum()); + EXPECT_EQ(merged.percentile(0), seq3.percentile(0)); for (int i = 0; i <= 100; i += 10) { - EXPECT_NEAR(seq2.percentile(i), seq3.percentile(i), err); + EXPECT_NEAR(merged.percentile(i), seq3.percentile(i), err); } - EXPECT_NEAR(seq2.avg(), seq3.avg(), err); - EXPECT_NEAR(seq2.sd(), seq3.sd(), err); + EXPECT_NEAR(merged.avg(), seq3.avg(), err); + EXPECT_NEAR(merged.sd(), seq3.sd(), err); + + // These are not implemented + EXPECT_TRUE(isnan(merged.davg())); + EXPECT_TRUE(isnan(merged.dvariance())); } From 6f71ef6092d257df51d45baf0640c8ab85318898 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Tue, 2 May 2023 00:51:46 +0000 Subject: [PATCH 227/254] Add generations to freeset Reviewed-by: ysr, wkemper --- .../gc/shenandoah/shenandoahConcurrentGC.cpp | 2 +- .../share/gc/shenandoah/shenandoahFreeSet.cpp | 941 +++++++++++++----- .../share/gc/shenandoah/shenandoahFreeSet.hpp | 170 +++- .../share/gc/shenandoah/shenandoahFullGC.cpp | 5 + .../gc/shenandoah/shenandoahGeneration.cpp | 3 + .../share/gc/shenandoah/shenandoahHeap.cpp | 6 + .../share/gc/shenandoah/shenandoahHeap.hpp | 19 +- .../gc/shenandoah/shenandoahHeap.inline.hpp | 5 + 8 files changed, 882 insertions(+), 269 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index 85b0a5c1d0a34..b6d05325599aa 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -1199,8 +1199,8 @@ void ShenandoahConcurrentGC::op_final_updaterefs() { Universe::verify(); } - heap->rebuild_free_set(true /*concurrent*/); heap->adjust_generation_sizes(); + heap->rebuild_free_set(true /*concurrent*/); } void ShenandoahConcurrentGC::op_final_roots() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index db955d5bd2d7e..3ac7507941132 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -38,55 +38,411 @@ #include "memory/resourceArea.hpp" #include "runtime/orderAccess.hpp" -ShenandoahFreeSet::ShenandoahFreeSet(ShenandoahHeap* heap, size_t max_regions) : - _heap(heap), - _mutator_free_bitmap(max_regions, mtGC), - _collector_free_bitmap(max_regions, mtGC), - _max(max_regions) + +ShenandoahSetsOfFree::ShenandoahSetsOfFree(size_t max_regions, ShenandoahFreeSet* free_set) : + _max(max_regions), + _free_set(free_set), + _region_size_bytes(ShenandoahHeapRegion::region_size_bytes()) { + _membership = NEW_C_HEAP_ARRAY(ShenandoahFreeMemoryType, max_regions, mtGC); clear_internal(); } -void ShenandoahFreeSet::increase_used(size_t num_bytes) { - shenandoah_assert_heaplocked(); - _used += num_bytes; +ShenandoahSetsOfFree::~ShenandoahSetsOfFree() { + FREE_C_HEAP_ARRAY(ShenandoahFreeMemoryType, _membership); +} + + +void ShenandoahSetsOfFree::clear_internal() { + for (size_t idx = 0; idx < _max; idx++) { + _membership[idx] = NotFree; + } - assert(_used <= _capacity, "must not use more than we have: used: " SIZE_FORMAT - ", capacity: " SIZE_FORMAT ", num_bytes: " SIZE_FORMAT, _used, _capacity, num_bytes); + for (size_t idx = 0; idx < NumFreeSets; idx++) { + _leftmosts[idx] = _max; + _rightmosts[idx] = 0; + _leftmosts_empty[idx] = _max; + _rightmosts_empty[idx] = 0; + _capacity_of[idx] = 0; + _used_by[idx] = 0; + } + + _left_to_right_bias[Mutator] = true; + _left_to_right_bias[Collector] = false; + _left_to_right_bias[OldCollector] = false; + + _region_counts[Mutator] = 0; + _region_counts[Collector] = 0; + _region_counts[OldCollector] = 0; + _region_counts[NotFree] = _max; } -bool ShenandoahFreeSet::is_mutator_free(size_t idx) const { - assert (idx < _max, "index is sane: " SIZE_FORMAT " < " SIZE_FORMAT " (left: " SIZE_FORMAT ", right: " SIZE_FORMAT ")", - idx, _max, _mutator_leftmost, _mutator_rightmost); - return _mutator_free_bitmap.at(idx); +void ShenandoahSetsOfFree::clear_all() { + clear_internal(); } -bool ShenandoahFreeSet::is_collector_free(size_t idx) const { - assert (idx < _max, "index is sane: " SIZE_FORMAT " < " SIZE_FORMAT " (left: " SIZE_FORMAT ", right: " SIZE_FORMAT ")", - idx, _max, _collector_leftmost, _collector_rightmost); - return _collector_free_bitmap.at(idx); +void ShenandoahSetsOfFree::increase_used(ShenandoahFreeMemoryType which_set, size_t bytes) { + assert (which_set > NotFree && which_set < NumFreeSets, "Set must correspond to a valid freeset"); + _used_by[which_set] += bytes; + assert (_used_by[which_set] <= _capacity_of[which_set], + "Must not use (" SIZE_FORMAT ") more than capacity (" SIZE_FORMAT ") after increase by " SIZE_FORMAT, + _used_by[which_set], _capacity_of[which_set], bytes); } -// This is a temporary solution to work around a shortcoming with the existing free set implementation. -// TODO: -// Remove this function after restructing FreeSet representation. A problem in the existing implementation is that old-gen -// regions are not considered to reside within the is_collector_free range. -// -HeapWord* ShenandoahFreeSet::allocate_with_old_affiliation(ShenandoahAllocRequest& req, bool& in_new_region) { - ShenandoahAffiliation affiliation = ShenandoahAffiliation::OLD_GENERATION; +inline void ShenandoahSetsOfFree::shrink_bounds_if_touched(ShenandoahFreeMemoryType set, size_t idx) { + if (idx == _leftmosts[set]) { + while ((_leftmosts[set] < _max) && !in_free_set(_leftmosts[set], set)) { + _leftmosts[set]++; + } + if (_leftmosts_empty[set] < _leftmosts[set]) { + // This gets us closer to where we need to be; we'll scan further when leftmosts_empty is requested. + _leftmosts_empty[set] = _leftmosts[set]; + } + } + if (idx == _rightmosts[set]) { + while (_rightmosts[set] > 0 && !in_free_set(_rightmosts[set], set)) { + _rightmosts[set]--; + } + if (_rightmosts_empty[set] > _rightmosts[set]) { + // This gets us closer to where we need to be; we'll scan further when rightmosts_empty is requested. + _rightmosts_empty[set] = _rightmosts[set]; + } + } +} - size_t rightmost = MAX2(_collector_rightmost, _mutator_rightmost); - size_t leftmost = MIN2(_collector_leftmost, _mutator_leftmost); +inline void ShenandoahSetsOfFree::expand_bounds_maybe(ShenandoahFreeMemoryType set, size_t idx, size_t region_capacity) { + if (region_capacity == _region_size_bytes) { + if (_leftmosts_empty[set] > idx) { + _leftmosts_empty[set] = idx; + } + if (_rightmosts_empty[set] < idx) { + _rightmosts_empty[set] = idx; + } + } + if (_leftmosts[set] > idx) { + _leftmosts[set] = idx; + } + if (_rightmosts[set] < idx) { + _rightmosts[set] = idx; + } +} - for (size_t c = rightmost + 1; c > leftmost; c--) { - // size_t is unsigned, need to dodge underflow when _leftmost = 0 - size_t idx = c - 1; - ShenandoahHeapRegion* r = _heap->get_region(idx); - if (r->affiliation() == affiliation && !r->is_humongous()) { - if (!r->is_cset() && !has_no_alloc_capacity(r)) { - HeapWord* result = try_allocate_in(r, req, in_new_region); - if (result != nullptr) { - return result; +void ShenandoahSetsOfFree::remove_from_free_sets(size_t idx) { + assert (idx < _max, "index is sane: " SIZE_FORMAT " < " SIZE_FORMAT, idx, _max); + ShenandoahFreeMemoryType orig_set = membership(idx); + assert (orig_set > NotFree && orig_set < NumFreeSets, "Cannot remove from free sets if not already free"); + _membership[idx] = NotFree; + shrink_bounds_if_touched(orig_set, idx); + + _region_counts[orig_set]--; + _region_counts[NotFree]++; +} + + +void ShenandoahSetsOfFree::make_free(size_t idx, ShenandoahFreeMemoryType which_set, size_t region_capacity) { + assert (idx < _max, "index is sane: " SIZE_FORMAT " < " SIZE_FORMAT, idx, _max); + assert (_membership[idx] == NotFree, "Cannot make free if already free"); + assert (which_set > NotFree && which_set < NumFreeSets, "selected free set must be valid"); + _membership[idx] = which_set; + _capacity_of[which_set] += region_capacity; + expand_bounds_maybe(which_set, idx, region_capacity); + + _region_counts[NotFree]--; + _region_counts[which_set]++; +} + +void ShenandoahSetsOfFree::move_to_set(size_t idx, ShenandoahFreeMemoryType new_set, size_t region_capacity) { + assert (idx < _max, "index is sane: " SIZE_FORMAT " < " SIZE_FORMAT, idx, _max); + assert ((new_set > NotFree) && (new_set < NumFreeSets), "New set must be valid"); + ShenandoahFreeMemoryType orig_set = _membership[idx]; + assert ((orig_set > NotFree) && (orig_set < NumFreeSets), "Cannot move free unless already free"); + // Expected transitions: + // During rebuild: Mutator => Collector + // Mutator empty => Collector + // During flip_to_gc: + // Mutator empty => Collector + // Mutator empty => Old Collector + assert (((region_capacity < _region_size_bytes) && (orig_set == Mutator) && (new_set == Collector)) || + ((region_capacity == _region_size_bytes) && (orig_set == Mutator) && (new_set == Collector || new_set == OldCollector)), + "Unexpected movement between sets"); + + _membership[idx] = new_set; + _capacity_of[orig_set] -= region_capacity; + shrink_bounds_if_touched(orig_set, idx); + + _capacity_of[new_set] += region_capacity; + expand_bounds_maybe(new_set, idx, region_capacity); + + _region_counts[orig_set]--; + _region_counts[new_set]++; +} + +inline ShenandoahFreeMemoryType ShenandoahSetsOfFree::membership(size_t idx) const { + assert (idx < _max, "index is sane: " SIZE_FORMAT " < " SIZE_FORMAT, idx, _max); + return _membership[idx]; +} + + // Returns true iff region idx is in the test_set free_set. Before returning true, asserts that the free + // set is not empty. Requires that test_set != NotFree or NumFreeSets. +inline bool ShenandoahSetsOfFree::in_free_set(size_t idx, ShenandoahFreeMemoryType test_set) const { + assert (idx < _max, "index is sane: " SIZE_FORMAT " < " SIZE_FORMAT, idx, _max); + if (_membership[idx] == test_set) { + assert (test_set == NotFree || _free_set->alloc_capacity(idx) > 0, "Free regions must have alloc capacity"); + return true; + } else { + return false; + } +} + +inline size_t ShenandoahSetsOfFree::leftmost(ShenandoahFreeMemoryType which_set) const { + assert (which_set > NotFree && which_set < NumFreeSets, "selected free set must be valid"); + size_t idx = _leftmosts[which_set]; + if (idx >= _max) { + return _max; + } else { + assert (in_free_set(idx, which_set), "left-most region must be free"); + return idx; + } +} + +inline size_t ShenandoahSetsOfFree::rightmost(ShenandoahFreeMemoryType which_set) const { + assert (which_set > NotFree && which_set < NumFreeSets, "selected free set must be valid"); + size_t idx = _rightmosts[which_set]; + assert ((_leftmosts[which_set] == _max) || in_free_set(idx, which_set), "right-most region must be free"); + return idx; +} + +size_t ShenandoahSetsOfFree::leftmost_empty(ShenandoahFreeMemoryType which_set) { + assert (which_set > NotFree && which_set < NumFreeSets, "selected free set must be valid"); + for (size_t idx = _leftmosts_empty[which_set]; idx < _max; idx++) { + if ((membership(idx) == which_set) && (_free_set->alloc_capacity(idx) == _region_size_bytes)) { + _leftmosts_empty[which_set] = idx; + return idx; + } + } + _leftmosts_empty[which_set] = _max; + _rightmosts_empty[which_set] = 0; + return _max; +} + +inline size_t ShenandoahSetsOfFree::rightmost_empty(ShenandoahFreeMemoryType which_set) { + assert (which_set > NotFree && which_set < NumFreeSets, "selected free set must be valid"); + for (intptr_t idx = _rightmosts_empty[which_set]; idx >= 0; idx--) { + if ((membership(idx) == which_set) && (_free_set->alloc_capacity(idx) == _region_size_bytes)) { + _rightmosts_empty[which_set] = idx; + return idx; + } + } + _leftmosts_empty[which_set] = _max; + _rightmosts_empty[which_set] = 0; + return 0; +} + +inline bool ShenandoahSetsOfFree::alloc_from_left_bias(ShenandoahFreeMemoryType which_set) { + assert (which_set > NotFree && which_set < NumFreeSets, "selected free set must be valid"); + return _left_to_right_bias[which_set]; +} + +void ShenandoahSetsOfFree::establish_alloc_bias(ShenandoahFreeMemoryType which_set) { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + shenandoah_assert_heaplocked(); + assert (which_set > NotFree && which_set < NumFreeSets, "selected free set must be valid"); + + size_t middle = (_leftmosts[which_set] + _rightmosts[which_set]) / 2; + size_t available_in_first_half = 0; + size_t available_in_second_half = 0; + + for (size_t index = _leftmosts[which_set]; index < middle; index++) { + if (in_free_set(index, which_set)) { + ShenandoahHeapRegion* r = heap->get_region(index); + available_in_first_half += r->free(); + } + } + for (size_t index = middle; index <= _rightmosts[which_set]; index++) { + if (in_free_set(index, which_set)) { + ShenandoahHeapRegion* r = heap->get_region(index); + available_in_second_half += r->free(); + } + } + + // We desire to first consume the sparsely distributed regions in order that the remaining regions are densely packed. + // Densely packing regions reduces the effort to search for a region that has sufficient memory to satisfy a new allocation + // request. Regions become sparsely distributed following a Full GC, which tends to slide all regions to the front of the + // heap rather than allowing survivor regions to remain at the high end of the heap where we intend for them to congregate. + + // TODO: In the future, we may modify Full GC so that it slides old objects to the end of the heap and young objects to the + // front of the heap. If this is done, we can always search survivor Collector and OldCollector regions right to left. + _left_to_right_bias[which_set] = (available_in_second_half > available_in_first_half); +} + +#ifdef ASSERT +void ShenandoahSetsOfFree::assert_bounds() { + + size_t leftmosts[NumFreeSets]; + size_t rightmosts[NumFreeSets]; + size_t empty_leftmosts[NumFreeSets]; + size_t empty_rightmosts[NumFreeSets]; + + for (int i = 0; i < NumFreeSets; i++) { + leftmosts[i] = _max; + empty_leftmosts[i] = _max; + rightmosts[i] = 0; + empty_rightmosts[i] = 0; + } + + for (size_t i = 0; i < _max; i++) { + ShenandoahFreeMemoryType set = membership(i); + switch (set) { + case NotFree: + break; + + case Mutator: + case Collector: + case OldCollector: + { + size_t capacity = _free_set->alloc_capacity(i); + bool is_empty = (capacity == _region_size_bytes); + assert(capacity > 0, "free regions must have allocation capacity"); + if (i < leftmosts[set]) { + leftmosts[set] = i; + } + if (is_empty && (i < empty_leftmosts[set])) { + empty_leftmosts[set] = i; + } + if (i > rightmosts[set]) { + rightmosts[set] = i; + } + if (is_empty && (i > empty_rightmosts[set])) { + empty_rightmosts[set] = i; + } + break; + } + + case NumFreeSets: + default: + ShouldNotReachHere(); + } + } + + // Performance invariants. Failing these would not break the free set, but performance would suffer. + assert (leftmost(Mutator) <= _max, "leftmost in bounds: " SIZE_FORMAT " < " SIZE_FORMAT, leftmost(Mutator), _max); + assert (rightmost(Mutator) < _max, "rightmost in bounds: " SIZE_FORMAT " < " SIZE_FORMAT, rightmost(Mutator), _max); + + assert (leftmost(Mutator) == _max || in_free_set(leftmost(Mutator), Mutator), + "leftmost region should be free: " SIZE_FORMAT, leftmost(Mutator)); + assert (leftmost(Mutator) == _max || in_free_set(rightmost(Mutator), Mutator), + "rightmost region should be free: " SIZE_FORMAT, rightmost(Mutator)); + + // If Mutator set is empty, leftmosts will both equal max, rightmosts will both equal zero. Likewise for empty region sets. + size_t beg_off = leftmosts[Mutator]; + size_t end_off = rightmosts[Mutator]; + assert (beg_off >= leftmost(Mutator), + "free regions before the leftmost: " SIZE_FORMAT ", bound " SIZE_FORMAT, beg_off, leftmost(Mutator)); + assert (end_off <= rightmost(Mutator), + "free regions past the rightmost: " SIZE_FORMAT ", bound " SIZE_FORMAT, end_off, rightmost(Mutator)); + + beg_off = empty_leftmosts[Mutator]; + end_off = empty_rightmosts[Mutator]; + assert (beg_off >= leftmost_empty(Mutator), + "free empty regions before the leftmost: " SIZE_FORMAT ", bound " SIZE_FORMAT, beg_off, leftmost_empty(Mutator)); + assert (end_off <= rightmost_empty(Mutator), + "free empty regions past the rightmost: " SIZE_FORMAT ", bound " SIZE_FORMAT, end_off, rightmost_empty(Mutator)); + + // Performance invariants. Failing these would not break the free set, but performance would suffer. + assert (leftmost(Collector) <= _max, "leftmost in bounds: " SIZE_FORMAT " < " SIZE_FORMAT, leftmost(Collector), _max); + assert (rightmost(Collector) < _max, "rightmost in bounds: " SIZE_FORMAT " < " SIZE_FORMAT, rightmost(Collector), _max); + + assert (leftmost(Collector) == _max || in_free_set(leftmost(Collector), Collector), + "leftmost region should be free: " SIZE_FORMAT, leftmost(Collector)); + assert (leftmost(Collector) == _max || in_free_set(rightmost(Collector), Collector), + "rightmost region should be free: " SIZE_FORMAT, rightmost(Collector)); + + // If Collector set is empty, leftmosts will both equal max, rightmosts will both equal zero. Likewise for empty region sets. + beg_off = leftmosts[Collector]; + end_off = rightmosts[Collector]; + assert (beg_off >= leftmost(Collector), + "free regions before the leftmost: " SIZE_FORMAT ", bound " SIZE_FORMAT, beg_off, leftmost(Collector)); + assert (end_off <= rightmost(Collector), + "free regions past the rightmost: " SIZE_FORMAT ", bound " SIZE_FORMAT, end_off, rightmost(Collector)); + + beg_off = empty_leftmosts[Collector]; + end_off = empty_rightmosts[Collector]; + assert (beg_off >= leftmost_empty(Collector), + "free empty regions before the leftmost: " SIZE_FORMAT ", bound " SIZE_FORMAT, beg_off, leftmost_empty(Collector)); + assert (end_off <= rightmost_empty(Collector), + "free empty regions past the rightmost: " SIZE_FORMAT ", bound " SIZE_FORMAT, end_off, rightmost_empty(Collector)); + + // Performance invariants. Failing these would not break the free set, but performance would suffer. + assert (leftmost(OldCollector) <= _max, "leftmost in bounds: " SIZE_FORMAT " < " SIZE_FORMAT, leftmost(OldCollector), _max); + assert (rightmost(OldCollector) < _max, "rightmost in bounds: " SIZE_FORMAT " < " SIZE_FORMAT, rightmost(OldCollector), _max); + + assert (leftmost(OldCollector) == _max || in_free_set(leftmost(OldCollector), OldCollector), + "leftmost region should be free: " SIZE_FORMAT, leftmost(OldCollector)); + assert (leftmost(OldCollector) == _max || in_free_set(rightmost(OldCollector), OldCollector), + "rightmost region should be free: " SIZE_FORMAT, rightmost(OldCollector)); + + // If OldCollector set is empty, leftmosts will both equal max, rightmosts will both equal zero. Likewise for empty region sets. + beg_off = leftmosts[OldCollector]; + end_off = rightmosts[OldCollector]; + assert (beg_off >= leftmost(OldCollector), + "free regions before the leftmost: " SIZE_FORMAT ", bound " SIZE_FORMAT, beg_off, leftmost(OldCollector)); + assert (end_off <= rightmost(OldCollector), + "free regions past the rightmost: " SIZE_FORMAT ", bound " SIZE_FORMAT, end_off, rightmost(OldCollector)); + + beg_off = empty_leftmosts[OldCollector]; + end_off = empty_rightmosts[OldCollector]; + assert (beg_off >= leftmost_empty(OldCollector), + "free empty regions before the leftmost: " SIZE_FORMAT ", bound " SIZE_FORMAT, beg_off, leftmost_empty(OldCollector)); + assert (end_off <= rightmost_empty(OldCollector), + "free empty regions past the rightmost: " SIZE_FORMAT ", bound " SIZE_FORMAT, end_off, rightmost_empty(OldCollector)); +} +#endif + + +ShenandoahFreeSet::ShenandoahFreeSet(ShenandoahHeap* heap, size_t max_regions) : + _heap(heap), + _free_sets(max_regions, this) +{ + clear_internal(); +} + +// This allocates from a region within the old_collector_set. If affiliation equals OLD, the allocation must be taken +// from a region that is_old(). Otherwise, affiliation should be FREE, in which case this will put a previously unaffiliated +// region into service. +HeapWord* ShenandoahFreeSet::allocate_old_with_affiliation(ShenandoahAffiliation affiliation, + ShenandoahAllocRequest& req, bool& in_new_region) { + shenandoah_assert_heaplocked(); + size_t rightmost = + (affiliation == ShenandoahAffiliation::FREE)? _free_sets.rightmost_empty(OldCollector): _free_sets.rightmost(OldCollector); + size_t leftmost = + (affiliation == ShenandoahAffiliation::FREE)? _free_sets.leftmost_empty(OldCollector): _free_sets.leftmost(OldCollector); + if (_free_sets.alloc_from_left_bias(OldCollector)) { + // This mode picks up stragglers left by a full GC + for (size_t idx = leftmost; idx <= rightmost; idx++) { + if (_free_sets.in_free_set(idx, OldCollector)) { + ShenandoahHeapRegion* r = _heap->get_region(idx); + assert(r->is_trash() || !r->is_affiliated() || r->is_old(), "old_collector_set region has bad affiliation"); + if (r->affiliation() == affiliation) { + HeapWord* result = try_allocate_in(r, req, in_new_region); + if (result != nullptr) { + return result; + } + } + } + } + } else { + // This mode picks up stragglers left by a previous concurrent GC + for (size_t count = rightmost + 1; count > leftmost; count--) { + // size_t is unsigned, need to dodge underflow when _leftmost = 0 + size_t idx = count - 1; + if (_free_sets.in_free_set(idx, OldCollector)) { + ShenandoahHeapRegion* r = _heap->get_region(idx); + assert(r->is_trash() || !r->is_affiliated() || r->is_old(), "old_collector_set region has bad affiliation"); + if (r->affiliation() == affiliation) { + HeapWord* result = try_allocate_in(r, req, in_new_region); + if (result != nullptr) { + return result; + } } } } @@ -95,10 +451,15 @@ HeapWord* ShenandoahFreeSet::allocate_with_old_affiliation(ShenandoahAllocReques } HeapWord* ShenandoahFreeSet::allocate_with_affiliation(ShenandoahAffiliation affiliation, ShenandoahAllocRequest& req, bool& in_new_region) { - for (size_t c = _collector_rightmost + 1; c > _collector_leftmost; c--) { + shenandoah_assert_heaplocked(); + size_t rightmost = + (affiliation == ShenandoahAffiliation::FREE)? _free_sets.rightmost_empty(Collector): _free_sets.rightmost(Collector); + size_t leftmost = + (affiliation == ShenandoahAffiliation::FREE)? _free_sets.leftmost_empty(Collector): _free_sets.leftmost(Collector); + for (size_t c = rightmost + 1; c > leftmost; c--) { // size_t is unsigned, need to dodge underflow when _leftmost = 0 size_t idx = c - 1; - if (is_collector_free(idx)) { + if (_free_sets.in_free_set(idx, Collector)) { ShenandoahHeapRegion* r = _heap->get_region(idx); if (r->affiliation() == affiliation) { HeapWord* result = try_allocate_in(r, req, in_new_region); @@ -113,6 +474,8 @@ HeapWord* ShenandoahFreeSet::allocate_with_affiliation(ShenandoahAffiliation aff } HeapWord* ShenandoahFreeSet::allocate_single(ShenandoahAllocRequest& req, bool& in_new_region) { + shenandoah_assert_heaplocked(); + // Scan the bitmap looking for a first fit. // // Leftmost and rightmost bounds provide enough caching to walk bitmap efficiently. Normally, @@ -157,9 +520,9 @@ HeapWord* ShenandoahFreeSet::allocate_single(ShenandoahAllocRequest& req, bool& case ShenandoahAllocRequest::_alloc_tlab: case ShenandoahAllocRequest::_alloc_shared: { // Try to allocate in the mutator view - for (size_t idx = _mutator_leftmost; idx <= _mutator_rightmost; idx++) { + for (size_t idx = _free_sets.leftmost(Mutator); idx <= _free_sets.rightmost(Mutator); idx++) { ShenandoahHeapRegion* r = _heap->get_region(idx); - if (is_mutator_free(idx) && (allow_new_region || r->is_affiliated())) { + if (_free_sets.in_free_set(idx, Mutator) && (allow_new_region || r->is_affiliated())) { // try_allocate_in() increases used if the allocation is successful. HeapWord* result = try_allocate_in(r, req, in_new_region); if (result != nullptr) { @@ -181,9 +544,9 @@ HeapWord* ShenandoahFreeSet::allocate_single(ShenandoahAllocRequest& req, bool& if (!_heap->mode()->is_generational()) { // size_t is unsigned, need to dodge underflow when _leftmost = 0 // Fast-path: try to allocate in the collector view first - for (size_t c = _collector_rightmost + 1; c > _collector_leftmost; c--) { + for (size_t c = _free_sets.rightmost(Collector) + 1; c > _free_sets.leftmost(Collector); c--) { size_t idx = c - 1; - if (is_collector_free(idx)) { + if (_free_sets.in_free_set(idx, Collector)) { HeapWord* result = try_allocate_in(_heap->get_region(idx), req, in_new_region); if (result != nullptr) { return result; @@ -194,9 +557,7 @@ HeapWord* ShenandoahFreeSet::allocate_single(ShenandoahAllocRequest& req, bool& // First try to fit into a region that is already in use in the same generation. HeapWord* result; if (req.is_old()) { - // TODO: this is a work around to address a deficiency in FreeSet representation. A better solution fixes - // the FreeSet implementation to deal more efficiently with old-gen regions as being in the "collector free set" - result = allocate_with_old_affiliation(req, in_new_region); + result = allocate_old_with_affiliation(req.affiliation(), req, in_new_region); } else { result = allocate_with_affiliation(req.affiliation(), req, in_new_region); } @@ -205,7 +566,11 @@ HeapWord* ShenandoahFreeSet::allocate_single(ShenandoahAllocRequest& req, bool& } if (allow_new_region) { // Then try a free region that is dedicated to GC allocations. - result = allocate_with_affiliation(FREE, req, in_new_region); + if (req.is_old()) { + result = allocate_old_with_affiliation(FREE, req, in_new_region); + } else { + result = allocate_with_affiliation(FREE, req, in_new_region); + } if (result != nullptr) { return result; } @@ -217,14 +582,38 @@ HeapWord* ShenandoahFreeSet::allocate_single(ShenandoahAllocRequest& req, bool& return nullptr; } + // TODO: + // if (!allow_new_region && req.is_old() && (young_generation->adjusted_unaffiliated_regions() > 0)) { + // transfer a region from young to old; + // allow_new_region = true; + // heap->set_old_evac_reserve(heap->get_old_evac_reserve() + region_size_bytes); + // } + // + // We should expand old-gen if this can prevent an old-gen evacuation failure. We don't care so much about + // promotion failures since they can be mitigated in a subsequent GC pass. Would be nice to know if this + // allocation request is for evacuation or promotion. Individual threads limit their use of PLAB memory for + // promotions, so we already have an assurance that any additional memory set aside for old-gen will be used + // only for old-gen evacuations. + + // Also TODO: + // if (GC is idle (out of cycle) and mutator allocation fails and there is memory reserved in Collector + // or OldCollector sets, transfer a region of memory so that we can satisfy the allocation request, and + // immediately trigger the start of GC. Is better to satisfy the allocation than to trigger out-of-cycle + // allocation failure (even if this means we have a little less memory to handle evacuations during the + // subsequent GC pass). + if (allow_new_region) { // Try to steal an empty region from the mutator view. - for (size_t c = _mutator_rightmost + 1; c > _mutator_leftmost; c--) { + for (size_t c = _free_sets.rightmost_empty(Mutator) + 1; c > _free_sets.leftmost_empty(Mutator); c--) { size_t idx = c - 1; - if (is_mutator_free(idx)) { + if (_free_sets.in_free_set(idx, Mutator)) { ShenandoahHeapRegion* r = _heap->get_region(idx); if (can_allocate_from(r)) { - flip_to_gc(r); + if (req.is_old()) { + flip_to_old_gc(r); + } else { + flip_to_gc(r); + } HeapWord *result = try_allocate_in(r, req, in_new_region); if (result != nullptr) { log_debug(gc, free)("Flipped region " SIZE_FORMAT " to gc for request: " PTR_FORMAT, idx, p2i(&req)); @@ -299,8 +688,7 @@ HeapWord* ShenandoahFreeSet::allocate_aligned_plab(size_t size, ShenandoahAllocR } HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, ShenandoahAllocRequest& req, bool& in_new_region) { - assert (!has_no_alloc_capacity(r), "Performance: should avoid full regions on this path: " SIZE_FORMAT, r->index()); - + assert (has_alloc_capacity(r), "Performance: should avoid full regions on this path: " SIZE_FORMAT, r->index()); if (_heap->is_concurrent_weak_root_in_progress() && r->is_trash()) { return nullptr; } @@ -391,7 +779,8 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah if (result != nullptr) { // Allocation successful, bump stats: if (req.is_mutator_alloc()) { - increase_used(req.actual_size() * HeapWordSize); + assert(req.is_young(), "Mutator allocations always come from young generation."); + _free_sets.increase_used(Mutator, req.actual_size() * HeapWordSize); } else { assert(req.is_gc_alloc(), "Should be gc_alloc since req wasn't mutator alloc"); @@ -410,69 +799,34 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah } } - if (result == nullptr || has_no_alloc_capacity(r)) { - // Region cannot afford this or future allocations. Retire it. + if (result == nullptr || alloc_capacity(r) < PLAB::min_size() * HeapWordSize) { + // Region cannot afford this and is likely to not afford future allocations. Retire it. // // While this seems a bit harsh, especially in the case when this large allocation does not - // fit, but the next small one would, we are risking to inflate scan times when lots of + // fit but the next small one would, we are risking to inflate scan times when lots of // almost-full regions precede the fully-empty region where we want to allocate the entire TLAB. - // TODO: Record first fully-empty region, and use that for large allocations and/or organize - // available free segments within regions for more efficient searches for "good fit". // Record the remainder as allocation waste + size_t idx = r->index(); if (req.is_mutator_alloc()) { size_t waste = r->free(); if (waste > 0) { - increase_used(waste); + _free_sets.increase_used(Mutator, waste); // This one request could cause several regions to be "retired", so we must accumulate the waste req.set_waste((waste >> LogHeapWordSize) + req.waste()); } + assert(_free_sets.membership(idx) == Mutator, "Must be mutator free: " SIZE_FORMAT, idx); + } else { + assert(_free_sets.membership(idx) == Collector || _free_sets.membership(idx) == OldCollector, + "Must be collector or old-collector free: " SIZE_FORMAT, idx); } - - size_t num = r->index(); - _collector_free_bitmap.clear_bit(num); - _mutator_free_bitmap.clear_bit(num); - // Touched the bounds? Need to update: - if (touches_bounds(num)) { - adjust_bounds(); - } - assert_bounds(); + // This region is no longer considered free (in any set) + _free_sets.remove_from_free_sets(idx); + _free_sets.assert_bounds(); } return result; } -bool ShenandoahFreeSet::touches_bounds(size_t num) const { - return num == _collector_leftmost || num == _collector_rightmost || num == _mutator_leftmost || num == _mutator_rightmost; -} - -void ShenandoahFreeSet::recompute_bounds() { - // Reset to the most pessimistic case: - _mutator_rightmost = _max - 1; - _mutator_leftmost = 0; - _collector_rightmost = _max - 1; - _collector_leftmost = 0; - - // ...and adjust from there - adjust_bounds(); -} - -void ShenandoahFreeSet::adjust_bounds() { - // Rewind both mutator bounds until the next bit. - while (_mutator_leftmost < _max && !is_mutator_free(_mutator_leftmost)) { - _mutator_leftmost++; - } - while (_mutator_rightmost > 0 && !is_mutator_free(_mutator_rightmost)) { - _mutator_rightmost--; - } - // Rewind both collector bounds until the next bit. - while (_collector_leftmost < _max && !is_collector_free(_collector_leftmost)) { - _collector_leftmost++; - } - while (_collector_rightmost > 0 && !is_collector_free(_collector_rightmost)) { - _collector_rightmost--; - } -} - HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req) { shenandoah_assert_heaplocked(); @@ -485,11 +839,11 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req) { // Check if there are enough regions left to satisfy allocation. if (_heap->mode()->is_generational()) { size_t avail_young_regions = generation->adjusted_unaffiliated_regions(); - if (num > mutator_count() || (num > avail_young_regions)) { + if (num > _free_sets.count(Mutator) || (num > avail_young_regions)) { return nullptr; } } else { - if (num > mutator_count()) { + if (num > _free_sets.count(Mutator)) { return nullptr; } } @@ -497,18 +851,18 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req) { // Find the continuous interval of $num regions, starting from $beg and ending in $end, // inclusive. Contiguous allocations are biased to the beginning. - size_t beg = _mutator_leftmost; + size_t beg = _free_sets.leftmost(Mutator); size_t end = beg; while (true) { - if (end >= _max) { + if (end >= _free_sets.max()) { // Hit the end, goodbye return nullptr; } // If regions are not adjacent, then current [beg; end] is useless, and we may fast-forward. // If region is not completely free, the current [beg; end] is useless, and we may fast-forward. - if (!is_mutator_free(end) || !can_allocate_from(_heap->get_region(end))) { + if (!_free_sets.in_free_set(end, Mutator) || !can_allocate_from(_heap->get_region(end))) { end++; beg = end; continue; @@ -552,17 +906,11 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req) { r->set_top(r->bottom() + used_words); // While individual regions report their true use, all humongous regions are marked used in the free set. - _mutator_free_bitmap.clear_bit(r->index()); + _free_sets.remove_from_free_sets(r->index()); } size_t total_humongous_size = ShenandoahHeapRegion::region_size_bytes() * num; - increase_used(total_humongous_size); - - // Allocated at left/rightmost? Move the bounds appropriately. - if (beg == _mutator_leftmost || end == _mutator_rightmost) { - adjust_bounds(); - } - assert_bounds(); - + _free_sets.increase_used(Mutator, total_humongous_size); + _free_sets.assert_bounds(); req.set_actual_size(words_size); if (remainder != 0) { req.set_waste(ShenandoahHeapRegion::region_size_words() - remainder); @@ -570,11 +918,19 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req) { return _heap->get_region(beg)->bottom(); } -bool ShenandoahFreeSet::can_allocate_from(ShenandoahHeapRegion *r) { +// Returns true iff this region is entirely available, either because it is empty() or because it has been found to represent +// immediate trash and we'll be able to immediately recycle it. Note that we cannot recycle immediate trash if +// concurrent weak root processing is in progress. +bool ShenandoahFreeSet::can_allocate_from(ShenandoahHeapRegion *r) const { return r->is_empty() || (r->is_trash() && !_heap->is_concurrent_weak_root_in_progress()); } -size_t ShenandoahFreeSet::alloc_capacity(ShenandoahHeapRegion *r) { +size_t ShenandoahFreeSet::alloc_capacity(size_t idx) const { + ShenandoahHeapRegion* r = _heap->get_region(idx); + return alloc_capacity(r); +} + +size_t ShenandoahFreeSet::alloc_capacity(ShenandoahHeapRegion *r) const { if (r->is_trash()) { // This would be recycled on allocation path return ShenandoahHeapRegion::region_size_bytes(); @@ -583,7 +939,16 @@ size_t ShenandoahFreeSet::alloc_capacity(ShenandoahHeapRegion *r) { } } -bool ShenandoahFreeSet::has_no_alloc_capacity(ShenandoahHeapRegion *r) { +bool ShenandoahFreeSet::has_alloc_capacity(ShenandoahHeapRegion *r) const { + return alloc_capacity(r) > 0; +} + +bool ShenandoahFreeSet::has_alloc_capacity(size_t idx) const { + ShenandoahHeapRegion* r = _heap->get_region(idx); + return alloc_capacity(r) > 0; +} + +bool ShenandoahFreeSet::has_no_alloc_capacity(ShenandoahHeapRegion *r) const { return alloc_capacity(r) == 0; } @@ -607,23 +972,30 @@ void ShenandoahFreeSet::recycle_trash() { } } -void ShenandoahFreeSet::flip_to_gc(ShenandoahHeapRegion* r) { +void ShenandoahFreeSet::flip_to_old_gc(ShenandoahHeapRegion* r) { size_t idx = r->index(); - assert(_mutator_free_bitmap.at(idx), "Should be in mutator view"); + assert(_free_sets.in_free_set(idx, Mutator), "Should be in mutator view"); assert(can_allocate_from(r), "Should not be allocated"); - _mutator_free_bitmap.clear_bit(idx); - _collector_free_bitmap.set_bit(idx); - _collector_leftmost = MIN2(idx, _collector_leftmost); - _collector_rightmost = MAX2(idx, _collector_rightmost); + size_t region_capacity = alloc_capacity(r); + _free_sets.move_to_set(idx, OldCollector, region_capacity); + _free_sets.assert_bounds(); - _capacity -= alloc_capacity(r); + // We do not ensure that the region is no longer trash, + // relying on try_allocate_in(), which always comes next, + // to recycle trash before attempting to allocate anything in the region. +} - if (touches_bounds(idx)) { - adjust_bounds(); - } - assert_bounds(); +void ShenandoahFreeSet::flip_to_gc(ShenandoahHeapRegion* r) { + size_t idx = r->index(); + + assert(_free_sets.in_free_set(idx, Mutator), "Should be in mutator view"); + assert(can_allocate_from(r), "Should not be allocated"); + + size_t region_capacity = alloc_capacity(r); + _free_sets.move_to_set(idx, Collector, region_capacity); + _free_sets.assert_bounds(); // We do not ensure that the region is no longer trash, // relying on try_allocate_in(), which always comes next, @@ -636,77 +1008,110 @@ void ShenandoahFreeSet::clear() { } void ShenandoahFreeSet::clear_internal() { - _mutator_free_bitmap.clear(); - _collector_free_bitmap.clear(); - _mutator_leftmost = _max; - _mutator_rightmost = 0; - _collector_leftmost = _max; - _collector_rightmost = 0; - _capacity = 0; - _used = 0; + _free_sets.clear_all(); } -void ShenandoahFreeSet::rebuild() { - shenandoah_assert_heaplocked(); - clear(); +// This function places all is_old() regions that have allocation capacity into the old_collector set. It places +// all other regions (not is_old()) that have allocation capacity into the mutator_set. Subsequently, we will +// move some of the mutator regions into the collector set or old_collector set with the intent of packing +// old_collector memory into the highest (rightmost) addresses of the heap and the collector memory into the +// next highest addresses of the heap, with mutator memory consuming the lowest addresses of the heap. +void ShenandoahFreeSet::find_regions_with_alloc_capacity() { - log_debug(gc, free)("Rebuilding FreeSet"); for (size_t idx = 0; idx < _heap->num_regions(); idx++) { ShenandoahHeapRegion* region = _heap->get_region(idx); if (region->is_alloc_allowed() || region->is_trash()) { - assert(!region->is_cset(), "Shouldn't be adding those to the free set"); - - // Do not add regions that would surely fail allocation - if (has_no_alloc_capacity(region)) continue; - - _capacity += alloc_capacity(region); - assert(_used <= _capacity, "must not use more than we have"); + assert(!region->is_cset(), "Shouldn't be adding cset regions to the free set"); + assert(_free_sets.in_free_set(idx, NotFree), "We are about to make region free; it should not be free already"); - assert(!is_mutator_free(idx), "We are about to add it, it shouldn't be there already"); - _mutator_free_bitmap.set_bit(idx); + // Do not add regions that would almost surely fail allocation + if (alloc_capacity(region) < PLAB::min_size() * HeapWordSize) continue; - log_debug(gc, free)(" Adding Region " SIZE_FORMAT " (Free: " SIZE_FORMAT "%s, Used: " SIZE_FORMAT "%s) to mutator free set", + if (region->is_old()) { + _free_sets.make_free(idx, OldCollector, alloc_capacity(region)); + log_debug(gc, free)( + " Adding Region " SIZE_FORMAT " (Free: " SIZE_FORMAT "%s, Used: " SIZE_FORMAT "%s) to old collector set", idx, byte_size_in_proper_unit(region->free()), proper_unit_for_byte_size(region->free()), - byte_size_in_proper_unit(region->used()), proper_unit_for_byte_size(region->used())); + byte_size_in_proper_unit(region->used()), proper_unit_for_byte_size(region->used())); + } else { + _free_sets.make_free(idx, Mutator, alloc_capacity(region)); + log_debug(gc, free)( + " Adding Region " SIZE_FORMAT " (Free: " SIZE_FORMAT "%s, Used: " SIZE_FORMAT "%s) to mutator set", + idx, byte_size_in_proper_unit(region->free()), proper_unit_for_byte_size(region->free()), + byte_size_in_proper_unit(region->used()), proper_unit_for_byte_size(region->used())); + } } } +} - // Evac reserve: reserve trailing space for evacuations +void ShenandoahFreeSet::rebuild() { + shenandoah_assert_heaplocked(); + // This resets all state information, removing all regions from all sets. + clear(); + + log_debug(gc, free)("Rebuilding FreeSet"); + + // This places regions that have alloc_capacity into the old_collector set if they identify as is_old() or the + // mutator set otherwise. + find_regions_with_alloc_capacity(); + + // Evac reserve: reserve trailing space for evacuations, with regions reserved for old evacuations placed to the right + // of regions reserved of young evacuations. + size_t young_reserve, old_reserve; if (!_heap->mode()->is_generational()) { - size_t to_reserve = (_heap->max_capacity() / 100) * ShenandoahEvacReserve; - reserve_regions(to_reserve); + young_reserve = (_heap->max_capacity() / 100) * ShenandoahEvacReserve; + old_reserve = 0; } else { - size_t young_reserve = (_heap->young_generation()->max_capacity() / 100) * ShenandoahEvacReserve; - // Note that all allocations performed from old-gen are performed by GC, generally using PLABs for both + // All allocations taken from the old collector set are performed by GC, generally using PLABs for both // promotions and evacuations. The partition between which old memory is reserved for evacuation and - // which is reserved for promotion is enforced using thread-local variables that prescribe intentons within - // each PLAB. We do not reserve any of old-gen memory in order to facilitate the loaning of old-gen memory - // to young-gen purposes. - size_t old_reserve = 0; - size_t to_reserve = young_reserve + old_reserve; - reserve_regions(to_reserve); + // which is reserved for promotion is enforced using thread-local variables that prescribe intentons for + // each PLAB's available memory. + if (_heap->has_evacuation_reserve_quantities()) { + // We are rebuilding at the end of final mark, having already established evacuation budgets for this GC pass. + young_reserve = _heap->get_young_evac_reserve(); + old_reserve = _heap->get_promoted_reserve() + _heap->get_old_evac_reserve(); + } else { + // We are rebuilding at end of GC, so we set aside budgets specified on command line (or defaults) + young_reserve = (_heap->young_generation()->max_capacity() * ShenandoahEvacReserve) / 100; + old_reserve = MAX2((_heap->old_generation()->max_capacity() * ShenandoahOldEvacReserve) / 100, + ShenandoahOldCompactionReserve * ShenandoahHeapRegion::region_size_bytes()); + } } + reserve_regions(young_reserve, old_reserve); - recompute_bounds(); - assert_bounds(); + _free_sets.establish_alloc_bias(OldCollector); + _free_sets.assert_bounds(); + log_status(); } -void ShenandoahFreeSet::reserve_regions(size_t to_reserve) { - size_t reserved = 0; - +// Having placed all regions that have allocation capacity into the mutator set if they identify as is_young() +// or into the old collector set if they identify as is_old(), move some of these regions from the mutator set +// into the collector set or old collector set in order to assure that the memory available for allocations within +// the collector set is at least to_reserve, and the memory available for allocations within the old collector set +// is at least to_reserve_old. +void ShenandoahFreeSet::reserve_regions(size_t to_reserve, size_t to_reserve_old) { for (size_t idx = _heap->num_regions() - 1; idx > 0; idx--) { - if (reserved >= to_reserve) break; - - ShenandoahHeapRegion* region = _heap->get_region(idx); - if (_mutator_free_bitmap.at(idx) && can_allocate_from(region)) { - _mutator_free_bitmap.clear_bit(idx); - _collector_free_bitmap.set_bit(idx); - size_t ac = alloc_capacity(region); - _capacity -= ac; - reserved += ac; - log_debug(gc, free)(" Shifting Region " SIZE_FORMAT " (Free: " SIZE_FORMAT "%s, Used: " SIZE_FORMAT "%s) to collector free set", - idx, byte_size_in_proper_unit(region->free()), proper_unit_for_byte_size(region->free()), - byte_size_in_proper_unit(region->used()), proper_unit_for_byte_size(region->used())); + ShenandoahHeapRegion* r = _heap->get_region(idx); + if (_free_sets.in_free_set(idx, Mutator)) { + assert (!r->is_old(), "mutator_is_free regions should not be affiliated OLD"); + size_t ac = alloc_capacity(r); + assert (ac > 0, "Membership in free set implies has capacity"); + + // OLD regions that have available memory are already in the old_collector free set + if ((_free_sets.capacity_of(OldCollector) < to_reserve_old) && (r->is_trash() || !r->is_affiliated())) { + _free_sets.move_to_set(idx, OldCollector, alloc_capacity(r)); + log_debug(gc, free)(" Shifting region " SIZE_FORMAT " from mutator_free to old_collector_free", idx); + } else if (_free_sets.capacity_of(Collector) < to_reserve) { + // Note: In a previous implementation, regions were only placed into the survivor space (collector_is_free) if + // they were entirely empty. I'm not sure I understand the rational for that. That alternative behavior would + // tend to mix survivor objects with ephemeral objects, making it more difficult to reclaim the memory for the + // ephemeral objects. It also delays aging of regions, causing promotion in place to be delayed. + _free_sets.move_to_set(idx, Collector, ac); + log_debug(gc)(" Shifting region " SIZE_FORMAT " from mutator_free to collector_free", idx); + } else { + // We've satisfied both to_reserve and to_reserved_old + break; + } } } } @@ -714,6 +1119,80 @@ void ShenandoahFreeSet::reserve_regions(size_t to_reserve) { void ShenandoahFreeSet::log_status() { shenandoah_assert_heaplocked(); +#ifdef ASSERT + // Dump of the FreeSet details is only enabled if assertions are enabled + { +#define BUFFER_SIZE 80 + size_t retired_old = 0; + size_t retired_old_humongous = 0; + size_t retired_young = 0; + size_t retired_young_humongous = 0; + size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); + char buffer[BUFFER_SIZE]; + for (uint i = 0; i < BUFFER_SIZE; i++) { + buffer[i] = '\0'; + } + log_info(gc, free)("FreeSet map legend:" + " M:mutator_free C:collector_free O:old_collector_free" + " H:humongous ~:retired old _:retired young"); + log_info(gc, free)(" mutator free range [" SIZE_FORMAT ".." SIZE_FORMAT "], " + " collector free range [" SIZE_FORMAT ".." SIZE_FORMAT "], " + "old collector free range [" SIZE_FORMAT ".." SIZE_FORMAT "] allocates from %s", + _free_sets.leftmost(Mutator), _free_sets.rightmost(Mutator), + _free_sets.leftmost(Collector), _free_sets.rightmost(Collector), + _free_sets.leftmost(OldCollector), _free_sets.rightmost(OldCollector), + _free_sets.alloc_from_left_bias(OldCollector)? "left to right": "right to left"); + + for (uint i = 0; i < _heap->num_regions(); i++) { + ShenandoahHeapRegion *r = _heap->get_region(i); + uint idx = i % 64; + if ((i != 0) && (idx == 0)) { + log_info(gc, free)(" %6u: %s", i-64, buffer); + } + if (_free_sets.in_free_set(i, Mutator)) { + assert(!r->is_old(), "Old regions should not be in mutator_free set"); + buffer[idx] = (alloc_capacity(r) == region_size_bytes)? 'M': 'm'; + } else if (_free_sets.in_free_set(i, Collector)) { + assert(!r->is_old(), "Old regions should not be in collector_free set"); + buffer[idx] = (alloc_capacity(r) == region_size_bytes)? 'C': 'c'; + } else if (_free_sets.in_free_set(i, OldCollector)) { + buffer[idx] = (alloc_capacity(r) == region_size_bytes)? 'O': 'o'; + } else if (r->is_humongous()) { + if (r->is_old()) { + buffer[idx] = 'H'; + retired_old_humongous += region_size_bytes; + } else { + buffer[idx] = 'h'; + retired_young_humongous += region_size_bytes; + } + } else { + if (r->is_old()) { + buffer[idx] = '~'; + retired_old += region_size_bytes; + } else { + buffer[idx] = '_'; + retired_young += region_size_bytes; + } + } + } + uint remnant = _heap->num_regions() % 64; + if (remnant > 0) { + buffer[remnant] = '\0'; + } else { + remnant = 64; + } + log_info(gc, free)(" %6u: %s", (uint) (_heap->num_regions() - remnant), buffer); + size_t total_young = retired_young + retired_young_humongous; + size_t total_old = retired_old + retired_old_humongous; + log_info(gc, free)("Retired young: " SIZE_FORMAT "%s (including humongous: " SIZE_FORMAT "%s), old: " SIZE_FORMAT + "%s (including humongous: " SIZE_FORMAT "%s)", + byte_size_in_proper_unit(total_young), proper_unit_for_byte_size(total_young), + byte_size_in_proper_unit(retired_young_humongous), proper_unit_for_byte_size(retired_young_humongous), + byte_size_in_proper_unit(total_old), proper_unit_for_byte_size(total_old), + byte_size_in_proper_unit(retired_old_humongous), proper_unit_for_byte_size(retired_old_humongous)); + } +#endif + LogTarget(Info, gc, free) lt; if (lt.is_enabled()) { ResourceMark rm; @@ -729,13 +1208,11 @@ void ShenandoahFreeSet::log_status() { size_t total_free = 0; size_t total_free_ext = 0; - for (size_t idx = _mutator_leftmost; idx <= _mutator_rightmost; idx++) { - if (is_mutator_free(idx)) { + for (size_t idx = _free_sets.leftmost(Mutator); idx <= _free_sets.rightmost(Mutator); idx++) { + if (_free_sets.in_free_set(idx, Mutator)) { ShenandoahHeapRegion *r = _heap->get_region(idx); size_t free = alloc_capacity(r); - max = MAX2(max, free); - if (r->is_empty()) { total_free_ext += free; if (last_idx + 1 == idx) { @@ -746,10 +1223,8 @@ void ShenandoahFreeSet::log_status() { } else { empty_contig = 0; } - total_used += r->used(); total_free += free; - max_contig = MAX2(max_contig, empty_contig); last_idx = idx; } @@ -758,6 +1233,10 @@ void ShenandoahFreeSet::log_status() { size_t max_humongous = max_contig * ShenandoahHeapRegion::region_size_bytes(); size_t free = capacity() - used(); + assert(free == total_free, "Sum of free within mutator regions (" SIZE_FORMAT + ") should match mutator capacity (" SIZE_FORMAT ") minus mutator used (" SIZE_FORMAT ")", + total_free, capacity(), used()); + ls.print("Free: " SIZE_FORMAT "%s, Max: " SIZE_FORMAT "%s regular, " SIZE_FORMAT "%s humongous, ", byte_size_in_proper_unit(total_free), proper_unit_for_byte_size(total_free), byte_size_in_proper_unit(max), proper_unit_for_byte_size(max), @@ -774,14 +1253,14 @@ void ShenandoahFreeSet::log_status() { ls.print(SIZE_FORMAT "%% external, ", frag_ext); size_t frag_int; - if (mutator_count() > 0) { - frag_int = (100 * (total_used / mutator_count()) / ShenandoahHeapRegion::region_size_bytes()); + if (_free_sets.count(Mutator) > 0) { + frag_int = (100 * (total_used / _free_sets.count(Mutator)) / ShenandoahHeapRegion::region_size_bytes()); } else { frag_int = 0; } ls.print(SIZE_FORMAT "%% internal; ", frag_int); - ls.print("Used: " SIZE_FORMAT "%s, Mutator Free: " SIZE_FORMAT " ", - byte_size_in_proper_unit(total_used), proper_unit_for_byte_size(total_used), mutator_count()); + ls.print("Used: " SIZE_FORMAT "%s, Mutator Free: " SIZE_FORMAT, + byte_size_in_proper_unit(total_used), proper_unit_for_byte_size(total_used), _free_sets.count(Mutator)); } { @@ -789,8 +1268,8 @@ void ShenandoahFreeSet::log_status() { size_t total_free = 0; size_t total_used = 0; - for (size_t idx = _collector_leftmost; idx <= _collector_rightmost; idx++) { - if (is_collector_free(idx)) { + for (size_t idx = _free_sets.leftmost(Collector); idx <= _free_sets.rightmost(Collector); idx++) { + if (_free_sets.in_free_set(idx, Collector)) { ShenandoahHeapRegion *r = _heap->get_region(idx); size_t free = alloc_capacity(r); max = MAX2(max, free); @@ -798,8 +1277,27 @@ void ShenandoahFreeSet::log_status() { total_used += r->used(); } } + ls.print(" Collector Reserve: " SIZE_FORMAT "%s, Max: " SIZE_FORMAT "%s; Used: " SIZE_FORMAT "%s", + byte_size_in_proper_unit(total_free), proper_unit_for_byte_size(total_free), + byte_size_in_proper_unit(max), proper_unit_for_byte_size(max), + byte_size_in_proper_unit(total_used), proper_unit_for_byte_size(total_used)); + } + + if (_heap->mode()->is_generational()) { + size_t max = 0; + size_t total_free = 0; + size_t total_used = 0; - ls.print_cr("Reserve: " SIZE_FORMAT "%s, Max: " SIZE_FORMAT "%s, Used: " SIZE_FORMAT "%s", + for (size_t idx = _free_sets.leftmost(OldCollector); idx <= _free_sets.rightmost(OldCollector); idx++) { + if (_free_sets.in_free_set(idx, OldCollector)) { + ShenandoahHeapRegion *r = _heap->get_region(idx); + size_t free = alloc_capacity(r); + max = MAX2(max, free); + total_free += free; + total_used += r->used(); + } + } + ls.print_cr(" Old Collector Reserve: " SIZE_FORMAT "%s, Max: " SIZE_FORMAT "%s; Used: " SIZE_FORMAT "%s", byte_size_in_proper_unit(total_free), proper_unit_for_byte_size(total_free), byte_size_in_proper_unit(max), proper_unit_for_byte_size(max), byte_size_in_proper_unit(total_used), proper_unit_for_byte_size(total_used)); @@ -809,7 +1307,7 @@ void ShenandoahFreeSet::log_status() { HeapWord* ShenandoahFreeSet::allocate(ShenandoahAllocRequest& req, bool& in_new_region) { shenandoah_assert_heaplocked(); - assert_bounds(); + _free_sets.assert_bounds(); // Allocation request is known to satisfy all memory budgeting constraints. if (req.size() > ShenandoahHeapRegion::humongous_threshold_words()) { @@ -837,8 +1335,8 @@ HeapWord* ShenandoahFreeSet::allocate(ShenandoahAllocRequest& req, bool& in_new_ size_t ShenandoahFreeSet::unsafe_peek_free() const { // Deliberately not locked, this method is unsafe when free set is modified. - for (size_t index = _mutator_leftmost; index <= _mutator_rightmost; index++) { - if (index < _max && is_mutator_free(index)) { + for (size_t index = _free_sets.leftmost(Mutator); index <= _free_sets.rightmost(Mutator); index++) { + if (index < _free_sets.max() && _free_sets.in_free_set(index, Mutator)) { ShenandoahHeapRegion* r = _heap->get_region(index); if (r->free() >= MinTLABSize) { return r->free(); @@ -851,18 +1349,26 @@ size_t ShenandoahFreeSet::unsafe_peek_free() const { } void ShenandoahFreeSet::print_on(outputStream* out) const { - out->print_cr("Mutator Free Set: " SIZE_FORMAT "", mutator_count()); - for (size_t index = _mutator_leftmost; index <= _mutator_rightmost; index++) { - if (is_mutator_free(index)) { + out->print_cr("Mutator Free Set: " SIZE_FORMAT "", _free_sets.count(Mutator)); + for (size_t index = _free_sets.leftmost(Mutator); index <= _free_sets.rightmost(Mutator); index++) { + if (_free_sets.in_free_set(index, Mutator)) { _heap->get_region(index)->print_on(out); } } - out->print_cr("Collector Free Set: " SIZE_FORMAT "", collector_count()); - for (size_t index = _collector_leftmost; index <= _collector_rightmost; index++) { - if (is_collector_free(index)) { + out->print_cr("Collector Free Set: " SIZE_FORMAT "", _free_sets.count(Collector)); + for (size_t index = _free_sets.leftmost(Collector); index <= _free_sets.rightmost(Collector); index++) { + if (_free_sets.in_free_set(index, Collector)) { _heap->get_region(index)->print_on(out); } } + if (_heap->mode()->is_generational()) { + out->print_cr("Old Collector Free Set: " SIZE_FORMAT "", _free_sets.count(OldCollector)); + for (size_t index = _free_sets.leftmost(OldCollector); index <= _free_sets.rightmost(OldCollector); index++) { + if (_free_sets.in_free_set(index, OldCollector)) { + _heap->get_region(index)->print_on(out); + } + } + } } /* @@ -891,8 +1397,8 @@ double ShenandoahFreeSet::internal_fragmentation() { double linear = 0; int count = 0; - for (size_t index = _mutator_leftmost; index <= _mutator_rightmost; index++) { - if (is_mutator_free(index)) { + for (size_t index = _free_sets.leftmost(Mutator); index <= _free_sets.rightmost(Mutator); index++) { + if (_free_sets.in_free_set(index, Mutator)) { ShenandoahHeapRegion* r = _heap->get_region(index); size_t used = r->used(); squared += used * used; @@ -929,8 +1435,8 @@ double ShenandoahFreeSet::external_fragmentation() { size_t free = 0; - for (size_t index = _mutator_leftmost; index <= _mutator_rightmost; index++) { - if (is_mutator_free(index)) { + for (size_t index = _free_sets.leftmost(Mutator); index <= _free_sets.rightmost(Mutator); index++) { + if (_free_sets.in_free_set(index, Mutator)) { ShenandoahHeapRegion* r = _heap->get_region(index); if (r->is_empty()) { free += ShenandoahHeapRegion::region_size_bytes(); @@ -955,30 +1461,3 @@ double ShenandoahFreeSet::external_fragmentation() { } } -#ifdef ASSERT -void ShenandoahFreeSet::assert_bounds() const { - // Performance invariants. Failing these would not break the free set, but performance - // would suffer. - assert (_mutator_leftmost <= _max, "leftmost in bounds: " SIZE_FORMAT " < " SIZE_FORMAT, _mutator_leftmost, _max); - assert (_mutator_rightmost < _max, "rightmost in bounds: " SIZE_FORMAT " < " SIZE_FORMAT, _mutator_rightmost, _max); - - assert (_mutator_leftmost == _max || is_mutator_free(_mutator_leftmost), "leftmost region should be free: " SIZE_FORMAT, _mutator_leftmost); - assert (_mutator_rightmost == 0 || is_mutator_free(_mutator_rightmost), "rightmost region should be free: " SIZE_FORMAT, _mutator_rightmost); - - size_t beg_off = _mutator_free_bitmap.find_first_set_bit(0); - size_t end_off = _mutator_free_bitmap.find_first_set_bit(_mutator_rightmost + 1); - assert (beg_off >= _mutator_leftmost, "free regions before the leftmost: " SIZE_FORMAT ", bound " SIZE_FORMAT, beg_off, _mutator_leftmost); - assert (end_off == _max, "free regions past the rightmost: " SIZE_FORMAT ", bound " SIZE_FORMAT, end_off, _mutator_rightmost); - - assert (_collector_leftmost <= _max, "leftmost in bounds: " SIZE_FORMAT " < " SIZE_FORMAT, _collector_leftmost, _max); - assert (_collector_rightmost < _max, "rightmost in bounds: " SIZE_FORMAT " < " SIZE_FORMAT, _collector_rightmost, _max); - - assert (_collector_leftmost == _max || is_collector_free(_collector_leftmost), "leftmost region should be free: " SIZE_FORMAT, _collector_leftmost); - assert (_collector_rightmost == 0 || is_collector_free(_collector_rightmost), "rightmost region should be free: " SIZE_FORMAT, _collector_rightmost); - - beg_off = _collector_free_bitmap.find_first_set_bit(0); - end_off = _collector_free_bitmap.find_first_set_bit(_collector_rightmost + 1); - assert (beg_off >= _collector_leftmost, "free regions before the leftmost: " SIZE_FORMAT ", bound " SIZE_FORMAT, beg_off, _collector_leftmost); - assert (end_off == _max, "free regions past the rightmost: " SIZE_FORMAT ", bound " SIZE_FORMAT, end_off, _collector_rightmost); -} -#endif diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp index a20b1f5873533..ef43c236eab37 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp @@ -29,30 +29,136 @@ #include "gc/shenandoah/shenandoahHeapRegionSet.hpp" #include "gc/shenandoah/shenandoahHeap.hpp" -class ShenandoahFreeSet : public CHeapObj { +enum ShenandoahFreeMemoryType : uint8_t { + NotFree, + Mutator, + Collector, + OldCollector, + NumFreeSets +}; + +class ShenandoahSetsOfFree { + private: - ShenandoahHeap* const _heap; - CHeapBitMap _mutator_free_bitmap; - CHeapBitMap _collector_free_bitmap; - size_t _max; + size_t _max; // The maximum number of heap regions + ShenandoahFreeSet* _free_set; + size_t _region_size_bytes; + ShenandoahFreeMemoryType* _membership; + size_t _leftmosts[NumFreeSets]; + size_t _rightmosts[NumFreeSets]; + size_t _leftmosts_empty[NumFreeSets]; + size_t _rightmosts_empty[NumFreeSets]; + size_t _capacity_of[NumFreeSets]; + size_t _used_by[NumFreeSets]; + bool _left_to_right_bias[NumFreeSets]; + size_t _region_counts[NumFreeSets]; + + inline void shrink_bounds_if_touched(ShenandoahFreeMemoryType set, size_t idx); + inline void expand_bounds_maybe(ShenandoahFreeMemoryType set, size_t idx, size_t capacity); + + // Restore all state variables to initial default state. + void clear_internal(); - // Left-most and right-most region indexes. There are no free regions outside - // of [left-most; right-most] index intervals - size_t _mutator_leftmost, _mutator_rightmost; - size_t _collector_leftmost, _collector_rightmost; +public: + ShenandoahSetsOfFree(size_t max_regions, ShenandoahFreeSet* free_set); + ~ShenandoahSetsOfFree(); + + // Make all regions NotFree and reset all bounds + void clear_all(); + + // Remove or retire region idx from all free sets. Requires that idx is in a free set. This does not affect capacity. + void remove_from_free_sets(size_t idx); + + // Place region idx into free set which_set. Requires that idx is currently NotFree. + void make_free(size_t idx, ShenandoahFreeMemoryType which_set, size_t region_capacity); + + // Place region idx into free set new_set. Requires that idx is currently not NotFRee. + void move_to_set(size_t idx, ShenandoahFreeMemoryType new_set, size_t region_capacity); + + // Returns the ShenandoahFreeMemoryType affiliation of region idx, or NotFree if this region is not currently free. This does + // not enforce that free_set membership implies allocation capacity. + inline ShenandoahFreeMemoryType membership(size_t idx) const; + + // Returns true iff region idx is in the test_set free_set. Before returning true, asserts that the free + // set is not empty. Requires that test_set != NotFree or NumFreeSets. + inline bool in_free_set(size_t idx, ShenandoahFreeMemoryType which_set) const; + + // The following four methods return the left-most and right-most bounds on ranges of regions representing + // the requested set. The _empty variants represent bounds on the range that holds completely empty + // regions, which are required for humongous allocations and desired for "very large" allocations. A + // return value of -1 from leftmost() or leftmost_empty() denotes that the corresponding set is empty. + // In other words: + // if the requested which_set is empty: + // leftmost() and leftmost_empty() return _max, rightmost() and rightmost_empty() return 0 + // otherwise, expect the following: + // 0 <= leftmost <= leftmost_empty <= rightmost_empty <= rightmost < _max + inline size_t leftmost(ShenandoahFreeMemoryType which_set) const; + inline size_t rightmost(ShenandoahFreeMemoryType which_set) const; + size_t leftmost_empty(ShenandoahFreeMemoryType which_set); + size_t rightmost_empty(ShenandoahFreeMemoryType which_set); + + inline void increase_used(ShenandoahFreeMemoryType which_set, size_t bytes); + + inline size_t capacity_of(ShenandoahFreeMemoryType which_set) const { + assert (which_set > NotFree && which_set < NumFreeSets, "selected free set must be valid"); + return _capacity_of[which_set]; + } - size_t _capacity; - size_t _used; + inline size_t used_by(ShenandoahFreeMemoryType which_set) const { + assert (which_set > NotFree && which_set < NumFreeSets, "selected free set must be valid"); + return _used_by[which_set]; + } - void assert_bounds() const NOT_DEBUG_RETURN; + inline size_t max() const { return _max; } + + inline size_t count(ShenandoahFreeMemoryType which_set) const { return _region_counts[which_set]; } + + // Return true iff regions for allocation from this set should be peformed left to right. Otherwise, allocate + // from right to left. + inline bool alloc_from_left_bias(ShenandoahFreeMemoryType which_set); + + // Determine whether we prefer to allocate from left to right or from right to left for this free-set. + void establish_alloc_bias(ShenandoahFreeMemoryType which_set); + + // Assure leftmost, rightmost, leftmost_empty, and rightmost_empty bounds are valid for all free sets. + // Valid bounds honor all of the following (where max is the number of heap regions): + // if the set is empty, leftmost equals max and rightmost equals 0 + // Otherwise (the set is not empty): + // 0 <= leftmost < max and 0 <= rightmost < max + // the region at leftmost is in the set + // the region at rightmost is in the set + // rightmost >= leftmost + // for every idx that is in the set { + // idx >= leftmost && + // idx <= rightmost + // } + // if the set has no empty regions, leftmost_empty equals max and rightmost_empty equals 0 + // Otherwise (the region has empty regions): + // 0 <= lefmost_empty < max and 0 <= rightmost_empty < max + // rightmost_empty >= leftmost_empty + // for every idx that is in the set and is empty { + // idx >= leftmost && + // idx <= rightmost + // } + void assert_bounds() NOT_DEBUG_RETURN; +}; - bool is_mutator_free(size_t idx) const; - bool is_collector_free(size_t idx) const; +class ShenandoahFreeSet : public CHeapObj { +private: + ShenandoahHeap* const _heap; + ShenandoahSetsOfFree _free_sets; HeapWord* try_allocate_in(ShenandoahHeapRegion* region, ShenandoahAllocRequest& req, bool& in_new_region); + HeapWord* allocate_aligned_plab(size_t size, ShenandoahAllocRequest& req, ShenandoahHeapRegion* r); + + // Satisfy young-generation or single-generation collector allocation request req by finding memory that matches + // affiliation, which either equals req.affiliation or FREE. We know req.is_young(). HeapWord* allocate_with_affiliation(ShenandoahAffiliation affiliation, ShenandoahAllocRequest& req, bool& in_new_region); - HeapWord* allocate_with_old_affiliation(ShenandoahAllocRequest& req, bool& in_new_region); + + // Satisfy allocation request req by finding memory that matches affiliation, which either equals req.affiliation + // or FREE. We know req.is_old(). + HeapWord* allocate_old_with_affiliation(ShenandoahAffiliation affiliation, ShenandoahAllocRequest& req, bool& in_new_region); // While holding the heap lock, allocate memory for a single object which is to be entirely contained // within a single HeapRegion as characterized by req. The req.size() value is known to be less than or @@ -64,29 +170,22 @@ class ShenandoahFreeSet : public CHeapObj { HeapWord* allocate_contiguous(ShenandoahAllocRequest& req); void flip_to_gc(ShenandoahHeapRegion* r); + void flip_to_old_gc(ShenandoahHeapRegion* r); - void recompute_bounds(); - void adjust_bounds(); - bool touches_bounds(size_t num) const; - - // Used of free set represents the amount of is_mutator_free set that has been consumed since most recent rebuild. - void increase_used(size_t amount); void clear_internal(); void try_recycle_trashed(ShenandoahHeapRegion *r); - bool can_allocate_from(ShenandoahHeapRegion *r); - size_t alloc_capacity(ShenandoahHeapRegion *r); - bool has_no_alloc_capacity(ShenandoahHeapRegion *r); + bool can_allocate_from(ShenandoahHeapRegion *r) const; + bool has_alloc_capacity(size_t idx) const; + bool has_alloc_capacity(ShenandoahHeapRegion *r) const; + bool has_no_alloc_capacity(ShenandoahHeapRegion *r) const; public: ShenandoahFreeSet(ShenandoahHeap* heap, size_t max_regions); - // Number of regions dedicated to GC allocations (for evacuation or promotion) that are currently free - size_t collector_count() const { return _collector_free_bitmap.count_one_bits(); } - - // Number of regions dedicated to mutator allocations that are currently free - size_t mutator_count() const { return _mutator_free_bitmap.count_one_bits(); } + size_t alloc_capacity(ShenandoahHeapRegion *r) const; + size_t alloc_capacity(size_t idx) const; void clear(); void rebuild(); @@ -95,11 +194,11 @@ class ShenandoahFreeSet : public CHeapObj { void log_status(); - size_t capacity() const { return _capacity; } - size_t used() const { return _used; } - size_t available() const { - assert(_used <= _capacity, "must use less than capacity"); - return _capacity - _used; + inline size_t capacity() const { return _free_sets.capacity_of(Mutator); } + inline size_t used() const { return _free_sets.used_by(Mutator); } + inline size_t available() const { + assert(used() <= capacity(), "must use less than capacity"); + return capacity() - used(); } HeapWord* allocate(ShenandoahAllocRequest& req, bool& in_new_region); @@ -110,7 +209,8 @@ class ShenandoahFreeSet : public CHeapObj { void print_on(outputStream* out) const; - void reserve_regions(size_t to_reserve); + void find_regions_with_alloc_capacity(); + void reserve_regions(size_t young_reserve, size_t old_reserve); }; #endif // SHARE_GC_SHENANDOAH_SHENANDOAHFREESET_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index f309dd5375da4..79c8b1c134520 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -1048,6 +1048,11 @@ void ShenandoahFullGC::distribute_slices(ShenandoahHeapRegionSet** worker_slices #endif } +// TODO: +// Consider compacting old-gen objects toward the high end of memory and young-gen objects towards the low-end +// of memory. As currently implemented, all regions are compacted toward the low-end of memory. This creates more +// fragmentation of the heap, because old-gen regions get scattered among low-address regions such that it becomes +// more difficult to find contiguous regions for humongous objects. void ShenandoahFullGC::phase2_calculate_target_addresses(ShenandoahHeapRegionSet** worker_slices) { GCTraceTime(Info, gc, phases) time("Phase 2: Compute new object addresses", _gc_timer); ShenandoahGCPhase calculate_address_phase(ShenandoahPhaseTimings::full_gc_calculate_addresses); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index bbdb6977822eb..8755883b2c50f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -833,12 +833,15 @@ void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) { } } + // Freeset construction uses reserve quantities if they are valid + heap->set_evacuation_reserve_quantities(true); { ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_rebuild_freeset : ShenandoahPhaseTimings::degen_gc_final_rebuild_freeset); ShenandoahHeapLocker locker(heap->lock()); heap->free_set()->rebuild(); } + heap->set_evacuation_reserve_quantities(false); } bool ShenandoahGeneration::is_bitmap_clear() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index b7ed4dda4d558..65385738ebecb 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -557,6 +557,8 @@ ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) : _young_evac_reserve(0), _captured_old_usage(0), _previous_promotion(0), + _upgraded_to_full(false), + _has_evacuation_reserve_quantities(false), _cancel_requested_time(0), _young_generation(nullptr), _global_generation(nullptr), @@ -2300,6 +2302,10 @@ void ShenandoahHeap::set_gc_state_mask(uint mask, bool value) { set_gc_state_all_threads(_gc_state.raw_value()); } +void ShenandoahHeap::set_evacuation_reserve_quantities(bool is_valid) { + _has_evacuation_reserve_quantities = is_valid; +} + void ShenandoahHeap::set_concurrent_young_mark_in_progress(bool in_progress) { if (has_forwarded_objects()) { set_gc_state_mask(YOUNG_MARKING | UPDATEREFS, in_progress); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index fecc41fb39008..a00a8a89f8839 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -379,15 +379,29 @@ class ShenandoahHeap : public CollectedHeap { bool _upgraded_to_full; + // At the end of final mark, but before we begin evacuating, heuristics calculate how much memory is required to + // hold the results of evacuating to young-gen and to old-gen. These quantitites, stored in _promoted_reserve, + // _old_evac_reserve, and _young_evac_reserve, are consulted prior to rebuilding the free set (ShenandoahFreeSet) + // in preparation for evacuation. When the free set is rebuilt, we make sure to reserve sufficient memory in the + // collector and old_collector sets to hold if _has_evacuation_reserve_quantities is true. The other time we + // rebuild the freeset is at the end of GC, as we prepare to idle GC until the next trigger. In this case, + // _has_evacuation_reserve_quantities is false because we don't yet know how much memory will need to be evacuated + // in the next GC cycle. When _has_evacuation_reserve_quantities is false, the free set rebuild operation reserves + // for the collector and old_collector sets based on alternative mechanisms, such as ShenandoahEvacReserve, + // ShenandoahOldEvacReserve, and ShenandoahOldCompactionReserve. In a future planned enhancement, the reserve + // for old_collector set when not _has_evacuation_reserve_quantities is based in part on anticipated promotion as + // determined by analysis of live data found during the previous GC pass which is one less than the current tenure age. + bool _has_evacuation_reserve_quantities; + void set_gc_state_all_threads(char state); void set_gc_state_mask(uint mask, bool value); - - public: + char gc_state() const; static address gc_state_addr(); + void set_evacuation_reserve_quantities(bool is_valid); void set_concurrent_young_mark_in_progress(bool in_progress); void set_concurrent_old_mark_in_progress(bool in_progress); void set_evacuation_in_progress(bool in_progress); @@ -403,6 +417,7 @@ class ShenandoahHeap : public CollectedHeap { inline bool is_stable() const; inline bool is_idle() const; + inline bool has_evacuation_reserve_quantities() const; inline bool is_concurrent_mark_in_progress() const; inline bool is_concurrent_young_mark_in_progress() const; inline bool is_concurrent_old_mark_in_progress() const; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp index 731b49f4cf6cc..2fbf1c0aad292 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp @@ -661,10 +661,15 @@ inline bool ShenandoahHeap::in_collection_set_loc(void* p) const { return collection_set()->is_in_loc(p); } + inline bool ShenandoahHeap::is_stable() const { return _gc_state.is_clear(); } +inline bool ShenandoahHeap::has_evacuation_reserve_quantities() const { + return _has_evacuation_reserve_quantities; +} + inline bool ShenandoahHeap::is_idle() const { return _gc_state.is_unset(YOUNG_MARKING | OLD_MARKING | EVACUATION | UPDATEREFS); } From 4e1ad181461cdaf736bcf622a619c95724ae0ca6 Mon Sep 17 00:00:00 2001 From: Aleksey Shipilev Date: Wed, 3 May 2023 13:29:47 +0000 Subject: [PATCH 228/254] Remove the remaining unclean upstream difference Reviewed-by: kdnilsen --- src/hotspot/share/gc/shared/ageTable.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/shared/ageTable.hpp b/src/hotspot/share/gc/shared/ageTable.hpp index 96250731ed27d..9f0c10ec31203 100644 --- a/src/hotspot/share/gc/shared/ageTable.hpp +++ b/src/hotspot/share/gc/shared/ageTable.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it From 90a1c9bfe1572c99c23bd63e686e34a1695c3bce Mon Sep 17 00:00:00 2001 From: William Kemper Date: Wed, 3 May 2023 17:08:49 +0000 Subject: [PATCH 229/254] Use distinct "end of cycle" message for each Shenandoah pause Reviewed-by: kdnilsen, ysr --- .../gc/shenandoah/shenandoahConcurrentGC.cpp | 25 ++++++++++++-- .../gc/shenandoah/shenandoahConcurrentGC.hpp | 2 +- .../share/gc/shenandoah/shenandoahOldGC.cpp | 2 +- .../share/gc/shenandoah/shenandoahUtils.cpp | 5 +-- .../share/gc/shenandoah/shenandoahUtils.hpp | 2 +- .../gc/shenandoah/shenandoahVMOperations.cpp | 33 ++++--------------- .../gc/shenandoah/shenandoahVMOperations.hpp | 6 ++-- src/hotspot/share/services/memoryManager.cpp | 5 +-- src/hotspot/share/services/memoryManager.hpp | 2 +- src/hotspot/share/services/memoryService.cpp | 16 +++++---- src/hotspot/share/services/memoryService.hpp | 9 +++-- .../mxbeans/TestPauseNotifications.java | 12 ++++++- 12 files changed, 68 insertions(+), 51 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index b6d05325599aa..0ce2fea576981 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -213,7 +213,7 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) { entry_cleanup_complete(); } else { // We chose not to evacuate because we found sufficient immediate garbage. - vmop_entry_final_roots(heap->is_aging_cycle()); + vmop_entry_final_roots(); _abbreviated = true; } @@ -283,14 +283,14 @@ void ShenandoahConcurrentGC::vmop_entry_final_updaterefs() { VMThread::execute(&op); } -void ShenandoahConcurrentGC::vmop_entry_final_roots(bool increment_region_ages) { +void ShenandoahConcurrentGC::vmop_entry_final_roots() { ShenandoahHeap* const heap = ShenandoahHeap::heap(); TraceCollectorStats tcs(heap->monitoring_support()->stw_collection_counters()); ShenandoahTimingsTracker timing(ShenandoahPhaseTimings::final_roots_gross); // This phase does not use workers, no need for setup heap->try_inject_alloc_failure(); - VM_ShenandoahFinalRoots op(this, increment_region_ages); + VM_ShenandoahFinalRoots op(this); VMThread::execute(&op); } @@ -1204,6 +1204,25 @@ void ShenandoahConcurrentGC::op_final_updaterefs() { } void ShenandoahConcurrentGC::op_final_roots() { + + ShenandoahHeap* heap = ShenandoahHeap::heap(); + if (heap->is_aging_cycle()) { + ShenandoahMarkingContext* ctx = heap->complete_marking_context(); + + for (size_t i = 0; i < heap->num_regions(); i++) { + ShenandoahHeapRegion *r = heap->get_region(i); + if (r->is_active() && r->is_young()) { + HeapWord* tams = ctx->top_at_mark_start(r); + HeapWord* top = r->top(); + if (top > tams) { + r->reset_age(); + } else { + r->increment_age(); + } + } + } + } + ShenandoahHeap::heap()->set_concurrent_weak_root_in_progress(false); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp index 7d0fb0ecbd969..32562de22621f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.hpp @@ -69,7 +69,7 @@ class ShenandoahConcurrentGC : public ShenandoahGC { protected: void vmop_entry_final_mark(); - void vmop_entry_final_roots(bool incr_region_ages); + void vmop_entry_final_roots(); private: void vmop_entry_init_updaterefs(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp index 5d84bd8e4aab8..94e29209bac5c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp @@ -144,7 +144,7 @@ bool ShenandoahOldGC::collect(GCCause::Cause cause) { // return from here with weak roots in progress. This is not a valid gc state // for any young collections (or allocation failures) that interrupt the old // collection. - vmop_entry_final_roots(false); + vmop_entry_final_roots(); return true; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp b/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp index 93658e6d44e61..4cb26b0c025df 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahUtils.cpp @@ -76,7 +76,7 @@ ShenandoahGCSession::~ShenandoahGCSession() { } -ShenandoahGCPauseMark::ShenandoahGCPauseMark(uint gc_id, SvcGCMarker::reason_type type) : +ShenandoahGCPauseMark::ShenandoahGCPauseMark(uint gc_id, const char* notification_message, SvcGCMarker::reason_type type) : _heap(ShenandoahHeap::heap()), _gc_id_mark(gc_id), _svc_gc_mark(type), _is_gc_active_mark() { _trace_pause.initialize(_heap->stw_memory_manager(), _heap->gc_cause(), /* allMemoryPoolsAffected */ true, @@ -86,7 +86,8 @@ ShenandoahGCPauseMark::ShenandoahGCPauseMark(uint gc_id, SvcGCMarker::reason_typ /* recordPostGCUsage = */ false, /* recordAccumulatedGCTime = */ true, /* recordGCEndTime = */ true, - /* countCollection = */ true + /* countCollection = */ true, + /* notificationMessage = */ notification_message ); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp b/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp index 60ae96077ab83..310af49fe56ed 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahUtils.hpp @@ -154,7 +154,7 @@ class ShenandoahGCPauseMark : public StackObj { TraceMemoryManagerStats _trace_pause; public: - ShenandoahGCPauseMark(uint gc_id, SvcGCMarker::reason_type type); + ShenandoahGCPauseMark(uint gc_id, const char* notification_action, SvcGCMarker::reason_type type); }; class ShenandoahSafepoint : public AllStatic { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp index 718d2f8766910..994ec5a3719cc 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp @@ -49,55 +49,36 @@ void VM_ShenandoahReferenceOperation::doit_epilogue() { } void VM_ShenandoahInitMark::doit() { - ShenandoahGCPauseMark mark(_gc_id, SvcGCMarker::CONCURRENT); + ShenandoahGCPauseMark mark(_gc_id, "Init Mark", SvcGCMarker::CONCURRENT); _gc->entry_init_mark(); } void VM_ShenandoahFinalMarkStartEvac::doit() { - ShenandoahGCPauseMark mark(_gc_id, SvcGCMarker::CONCURRENT); + ShenandoahGCPauseMark mark(_gc_id, "Final Mark", SvcGCMarker::CONCURRENT); _gc->entry_final_mark(); } void VM_ShenandoahFullGC::doit() { - ShenandoahGCPauseMark mark(_gc_id, SvcGCMarker::FULL); + ShenandoahGCPauseMark mark(_gc_id, "Full GC", SvcGCMarker::FULL); _full_gc->entry_full(_gc_cause); } void VM_ShenandoahDegeneratedGC::doit() { - ShenandoahGCPauseMark mark(_gc_id, SvcGCMarker::CONCURRENT); + ShenandoahGCPauseMark mark(_gc_id, "Degenerated GC", SvcGCMarker::CONCURRENT); _gc->entry_degenerated(); } void VM_ShenandoahInitUpdateRefs::doit() { - ShenandoahGCPauseMark mark(_gc_id, SvcGCMarker::CONCURRENT); + ShenandoahGCPauseMark mark(_gc_id, "Init Update Refs", SvcGCMarker::CONCURRENT); _gc->entry_init_updaterefs(); } void VM_ShenandoahFinalUpdateRefs::doit() { - ShenandoahGCPauseMark mark(_gc_id, SvcGCMarker::CONCURRENT); + ShenandoahGCPauseMark mark(_gc_id, "Final Update Refs", SvcGCMarker::CONCURRENT); _gc->entry_final_updaterefs(); } void VM_ShenandoahFinalRoots::doit() { - ShenandoahGCPauseMark mark(_gc_id, SvcGCMarker::CONCURRENT); - if (_incr_region_ages) { - // TODO: Do we even care about this? Do we want to parallelize it? - ShenandoahHeap* heap = ShenandoahHeap::heap(); - ShenandoahMarkingContext* ctx = heap->complete_marking_context(); - - for (size_t i = 0; i < heap->num_regions(); i++) { - ShenandoahHeapRegion *r = heap->get_region(i); - if (r->is_active() && r->is_young()) { - HeapWord* tams = ctx->top_at_mark_start(r); - HeapWord* top = r->top(); - if (top > tams) { - r->reset_age(); - } else if (heap->is_aging_cycle()) { - // TODO: Does _incr_region_ages imply heap->is_aging_cycle()? - r->increment_age(); - } - } - } - } + ShenandoahGCPauseMark mark(_gc_id, "Final Roots", SvcGCMarker::CONCURRENT); _gc->entry_final_roots(); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp index 716e4d9566e82..df7d6e685b9a3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp @@ -133,12 +133,10 @@ class VM_ShenandoahFinalUpdateRefs: public VM_ShenandoahOperation { class VM_ShenandoahFinalRoots: public VM_ShenandoahOperation { ShenandoahConcurrentGC* const _gc; - const bool _incr_region_ages; public: - VM_ShenandoahFinalRoots(ShenandoahConcurrentGC* gc, bool incr_region_ages) : + VM_ShenandoahFinalRoots(ShenandoahConcurrentGC* gc) : VM_ShenandoahOperation(), - _gc(gc), - _incr_region_ages(incr_region_ages) {}; + _gc(gc) {}; VM_Operation::VMOp_Type type() const { return VMOp_ShenandoahFinalRoots; } const char* name() const { return "Shenandoah Final Roots"; } virtual void doit(); diff --git a/src/hotspot/share/services/memoryManager.cpp b/src/hotspot/share/services/memoryManager.cpp index d485184017526..73b3bde5b0f5a 100644 --- a/src/hotspot/share/services/memoryManager.cpp +++ b/src/hotspot/share/services/memoryManager.cpp @@ -243,7 +243,7 @@ void GCMemoryManager::gc_end(bool recordPostGCUsage, bool recordAccumulatedGCTime, bool recordGCEndTime, bool countCollection, GCCause::Cause cause, - bool allMemoryPoolsAffected) { + bool allMemoryPoolsAffected, const char* notificationMessage) { if (recordAccumulatedGCTime) { _accumulated_timer.stop(); } @@ -293,7 +293,8 @@ void GCMemoryManager::gc_end(bool recordPostGCUsage, } if (is_notification_enabled()) { - GCNotifier::pushNotification(this, _gc_end_message, GCCause::to_string(cause)); + const char* message = notificationMessage != nullptr ? notificationMessage : _gc_end_message; + GCNotifier::pushNotification(this, message, GCCause::to_string(cause)); } } } diff --git a/src/hotspot/share/services/memoryManager.hpp b/src/hotspot/share/services/memoryManager.hpp index ec40a9fc0d2f6..9cd7d66fff455 100644 --- a/src/hotspot/share/services/memoryManager.hpp +++ b/src/hotspot/share/services/memoryManager.hpp @@ -167,7 +167,7 @@ class GCMemoryManager : public MemoryManager { bool recordAccumulatedGCTime); void gc_end(bool recordPostGCUsage, bool recordAccumulatedGCTime, bool recordGCEndTime, bool countCollection, GCCause::Cause cause, - bool allMemoryPoolsAffected); + bool allMemoryPoolsAffected, const char* notificationMessage = nullptr); void reset_gc_stat() { _num_collections = 0; _accumulated_timer.reset(); } diff --git a/src/hotspot/share/services/memoryService.cpp b/src/hotspot/share/services/memoryService.cpp index 4391a8c2611e6..c0c877d02c922 100644 --- a/src/hotspot/share/services/memoryService.cpp +++ b/src/hotspot/share/services/memoryService.cpp @@ -183,10 +183,10 @@ void MemoryService::gc_end(GCMemoryManager* manager, bool recordPostGCUsage, bool recordAccumulatedGCTime, bool recordGCEndTime, bool countCollection, GCCause::Cause cause, - bool allMemoryPoolsAffected) { + bool allMemoryPoolsAffected, const char* notificationMessage) { // register the GC end statistics and memory usage manager->gc_end(recordPostGCUsage, recordAccumulatedGCTime, recordGCEndTime, - countCollection, cause, allMemoryPoolsAffected); + countCollection, cause, allMemoryPoolsAffected, notificationMessage); } bool MemoryService::set_verbose(bool verbose) { @@ -227,11 +227,12 @@ TraceMemoryManagerStats::TraceMemoryManagerStats(GCMemoryManager* gc_memory_mana bool recordPostGCUsage, bool recordAccumulatedGCTime, bool recordGCEndTime, - bool countCollection) { + bool countCollection, + const char* notificationMessage) { initialize(gc_memory_manager, cause, allMemoryPoolsAffected, recordGCBeginTime, recordPreGCUsage, recordPeakUsage, recordPostGCUsage, recordAccumulatedGCTime, recordGCEndTime, - countCollection); + countCollection, notificationMessage); } // for a subclass to create then initialize an instance before invoking @@ -245,7 +246,8 @@ void TraceMemoryManagerStats::initialize(GCMemoryManager* gc_memory_manager, bool recordPostGCUsage, bool recordAccumulatedGCTime, bool recordGCEndTime, - bool countCollection) { + bool countCollection, + const char* notificationMessage) { _gc_memory_manager = gc_memory_manager; _allMemoryPoolsAffected = allMemoryPoolsAffected; _recordGCBeginTime = recordGCBeginTime; @@ -256,6 +258,7 @@ void TraceMemoryManagerStats::initialize(GCMemoryManager* gc_memory_manager, _recordGCEndTime = recordGCEndTime; _countCollection = countCollection; _cause = cause; + _notificationMessage = notificationMessage; MemoryService::gc_begin(_gc_memory_manager, _recordGCBeginTime, _recordAccumulatedGCTime, _recordPreGCUsage, _recordPeakUsage); @@ -263,5 +266,6 @@ void TraceMemoryManagerStats::initialize(GCMemoryManager* gc_memory_manager, TraceMemoryManagerStats::~TraceMemoryManagerStats() { MemoryService::gc_end(_gc_memory_manager, _recordPostGCUsage, _recordAccumulatedGCTime, - _recordGCEndTime, _countCollection, _cause, _allMemoryPoolsAffected); + _recordGCEndTime, _countCollection, _cause, _allMemoryPoolsAffected, + _notificationMessage); } diff --git a/src/hotspot/share/services/memoryService.hpp b/src/hotspot/share/services/memoryService.hpp index 3c7bc483dfbed..7a64af3116b43 100644 --- a/src/hotspot/share/services/memoryService.hpp +++ b/src/hotspot/share/services/memoryService.hpp @@ -104,7 +104,7 @@ class MemoryService : public AllStatic { bool recordAccumulatedGCTime, bool recordGCEndTime, bool countCollection, GCCause::Cause cause, - bool allMemoryPoolsAffected); + bool allMemoryPoolsAffected, const char* notificationMessage = nullptr); static bool get_verbose() { return log_is_enabled(Info, gc); } static bool set_verbose(bool verbose); @@ -125,6 +125,7 @@ class TraceMemoryManagerStats : public StackObj { bool _recordGCEndTime; bool _countCollection; GCCause::Cause _cause; + const char* _notificationMessage; public: TraceMemoryManagerStats() {} TraceMemoryManagerStats(GCMemoryManager* gc_memory_manager, @@ -136,7 +137,8 @@ class TraceMemoryManagerStats : public StackObj { bool recordPostGCUsage = true, bool recordAccumulatedGCTime = true, bool recordGCEndTime = true, - bool countCollection = true); + bool countCollection = true, + const char* notificationMessage = nullptr); void initialize(GCMemoryManager* gc_memory_manager, GCCause::Cause cause, @@ -147,7 +149,8 @@ class TraceMemoryManagerStats : public StackObj { bool recordPostGCUsage, bool recordAccumulatedGCTime, bool recordGCEndTime, - bool countCollection); + bool countCollection, + const char* notificationMessage = nullptr); ~TraceMemoryManagerStats(); }; diff --git a/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestPauseNotifications.java b/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestPauseNotifications.java index 7c8407bd361d0..bef44298bbc4f 100644 --- a/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestPauseNotifications.java +++ b/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestPauseNotifications.java @@ -127,6 +127,12 @@ public class TestPauseNotifications { static volatile Object sink; + private static boolean isExpectedPauseAction(String action) { + return "Init Mark".equals(action) || "Final Mark".equals(action) || "Full GC".equals(action) + || "Degenerated GC".equals(action) || "Init Update Refs".equals(action) + || "Final Update Refs".equals(action) || "Final Roots".equals(action); + } + public static void main(String[] args) throws Exception { final long startTime = System.currentTimeMillis(); @@ -141,7 +147,8 @@ public void handleNotification(Notification n, Object o) { if (n.getType().equals(GarbageCollectionNotificationInfo.GARBAGE_COLLECTION_NOTIFICATION)) { GarbageCollectionNotificationInfo info = GarbageCollectionNotificationInfo.from((CompositeData) n.getUserData()); - System.out.println("Received: " + info.getGcName()); + System.out.println("Received: " + info.getGcName() + "/" + info.getGcAction()); + long d = info.getGcInfo().getDuration(); @@ -150,6 +157,9 @@ public void handleNotification(Notification n, Object o) { if (name.equals("Shenandoah Pauses")) { pausesCount.incrementAndGet(); pausesDuration.addAndGet(d); + if (!isExpectedPauseAction(info.getGcAction())) { + throw new IllegalStateException("Unknown action: " + info.getGcAction()); + } } else if (name.equals("Shenandoah Cycles")) { cyclesCount.incrementAndGet(); cyclesDuration.addAndGet(d); From 84c36e65a78c7f41adb61e5c2259340bdb9767d0 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Wed, 3 May 2023 17:16:52 +0000 Subject: [PATCH 230/254] Alphabetize includes Reviewed-by: shade --- src/hotspot/share/gc/shenandoah/shenandoahCardTable.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCardTable.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCardTable.cpp index d26173431cc79..1c76847ea68cd 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCardTable.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCardTable.cpp @@ -26,8 +26,8 @@ #include "gc/shenandoah/shenandoahCardTable.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahUtils.hpp" -#include "services/memTracker.hpp" #include "runtime/init.hpp" +#include "services/memTracker.hpp" void ShenandoahCardTable::initialize() { size_t num_cards = cards_required(_whole_heap.word_size()); From c966d9acccda5669f61f2d4a5e0a1557fcfcf372 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Mon, 8 May 2023 17:12:52 +0000 Subject: [PATCH 231/254] Remove passing tests from ProblemList.txt Reviewed-by: ysr, shade --- test/hotspot/jtreg/ProblemList.txt | 3 --- test/jdk/ProblemList.txt | 1 - 2 files changed, 4 deletions(-) diff --git a/test/hotspot/jtreg/ProblemList.txt b/test/hotspot/jtreg/ProblemList.txt index feb19de71e7fe..3392452f42391 100644 --- a/test/hotspot/jtreg/ProblemList.txt +++ b/test/hotspot/jtreg/ProblemList.txt @@ -86,9 +86,6 @@ gc/stress/TestStressG1Humongous.java 8286554 windows-x64 gc/shenandoah/TestDynamicSoftMaxHeapSize.java#generational 8306333 generic-all gc/shenandoah/oom/TestThreadFailure.java 8306335 generic-all gc/shenandoah/oom/TestClassLoaderLeak.java 8306336 generic-all -gc/shenandoah/mxbeans/TestChurnNotifications.java#generational 8306337 generic-all -gc/stress/gcold/TestGCOldWithShenandoah.java#generational 8306339 generic-all -gc/stress/systemgc/TestSystemGCWithShenandoah.java#generational 8306340 generic-all gc/stress/gclocker/TestGCLockerWithShenandoah.java#generational 8306341 generic-all gc/TestAllocHumongousFragment.java#generational 8306342 generic-all diff --git a/test/jdk/ProblemList.txt b/test/jdk/ProblemList.txt index 966c0f6c4619b..34f3393440be1 100644 --- a/test/jdk/ProblemList.txt +++ b/test/jdk/ProblemList.txt @@ -484,7 +484,6 @@ java/lang/invoke/LFCaching/LFMultiThreadCachingTest.java 8151492 generic- java/lang/invoke/LFCaching/LFGarbageCollectedTest.java 8078602 generic-all java/lang/invoke/lambda/LambdaFileEncodingSerialization.java 8249079 linux-x64 java/lang/invoke/RicochetTest.java 8251969 generic-all -java/lang/Thread/virtual/TraceVirtualThreadLocals.java 8306343 generic-all ############################################################################ From f969d53de1e9e318357d0f04b6270d3c9d012138 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Wed, 10 May 2023 15:55:54 +0000 Subject: [PATCH 232/254] Increase card offset mask Reviewed-by: kdnilsen --- src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp | 3 ++- test/hotspot/jtreg/ProblemList.txt | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp index 0c92a5eceeb47..6c4bf93747f1a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp @@ -391,13 +391,14 @@ class ShenandoahCardCluster: public CHeapObj { // a particular card region. We pack this bit into start byte under assumption that start byte is accessed less // frequently than last byte. This is true when number of clean cards is greater than number of dirty cards. static const uint16_t ObjectStartsInCardRegion = 0x80; - static const uint16_t FirstStartBits = 0x3f; + static const uint16_t FirstStartBits = 0x7f; crossing_info *object_starts; public: // If we're setting first_start, assume the card has an object. inline void set_first_start(size_t card_index, uint8_t value) { + assert(value < FirstStartBits, "Offset into card would be truncated"); object_starts[card_index].offsets.first = ObjectStartsInCardRegion | value; } diff --git a/test/hotspot/jtreg/ProblemList.txt b/test/hotspot/jtreg/ProblemList.txt index e056078c372a1..feb579ead6a5f 100644 --- a/test/hotspot/jtreg/ProblemList.txt +++ b/test/hotspot/jtreg/ProblemList.txt @@ -83,11 +83,11 @@ gc/stress/gclocker/TestGCLockerWithG1.java 8180622 generic-all gc/stress/TestJNIBlockFullGC/TestJNIBlockFullGC.java 8192647 generic-all gc/stress/TestStressG1Humongous.java 8286554 windows-x64 -gc/shenandoah/TestDynamicSoftMaxHeapSize.java#generational 8306333 generic-all gc/shenandoah/oom/TestThreadFailure.java 8306335 generic-all gc/shenandoah/oom/TestClassLoaderLeak.java 8306336 generic-all +gc/shenandoah/TestDynamicSoftMaxHeapSize.java#generational 8306333 generic-all gc/stress/gclocker/TestGCLockerWithShenandoah.java#generational 8306341 generic-all -gc/TestAllocHumongousFragment.java#generational 8306342 generic-all + ############################################################################# From a2ab9979c6fa12e8e7f1feadca248eb31eae07de Mon Sep 17 00:00:00 2001 From: William Kemper Date: Wed, 10 May 2023 22:48:11 +0000 Subject: [PATCH 233/254] Assert that region usage accounting is only accessed in generational mode Reviewed-by: kdnilsen --- src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp index 6c8acd1ec5fe2..c65956858b626 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGlobalGeneration.cpp @@ -30,7 +30,6 @@ #include "gc/shenandoah/shenandoahUtils.hpp" #include "gc/shenandoah/shenandoahVerifier.hpp" #include "gc/shenandoah/shenandoahYoungGeneration.hpp" -#include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp" const char* ShenandoahGlobalGeneration::name() const { return "GLOBAL"; @@ -42,6 +41,7 @@ size_t ShenandoahGlobalGeneration::max_capacity() const { size_t ShenandoahGlobalGeneration::used_regions() const { ShenandoahHeap* heap = ShenandoahHeap::heap(); + assert(heap->mode()->is_generational(), "Region usage accounting is only for generational mode"); return heap->old_generation()->used_regions() + heap->young_generation()->used_regions(); } From e48977a7bca71d7bf1d337bb7811f2719ceb3341 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Mon, 15 May 2023 17:08:56 +0000 Subject: [PATCH 234/254] Use static assert to validate card offset encoding mask Reviewed-by: ysr --- .../share/gc/shenandoah/shenandoahScanRemembered.hpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp index 6c4bf93747f1a..a11619e4d66c3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahScanRemembered.hpp @@ -183,6 +183,7 @@ #include "gc/shenandoah/shenandoahNumberSeq.hpp" #include "gc/shenandoah/shenandoahTaskqueue.hpp" #include "memory/iterator.hpp" +#include "utilities/globalDefinitions.hpp" class ShenandoahReferenceProcessor; class ShenandoahConcurrentMark; @@ -393,12 +394,16 @@ class ShenandoahCardCluster: public CHeapObj { static const uint16_t ObjectStartsInCardRegion = 0x80; static const uint16_t FirstStartBits = 0x7f; + // Check that we have enough bits to store the largest possible offset into a card for an object start. + // The value for maximum card size is based on the constraints for GCCardSizeInBytes in gc_globals.hpp. + static const int MaxCardSize = NOT_LP64(512) LP64_ONLY(1024); + STATIC_ASSERT((MaxCardSize / HeapWordSize) - 1 <= FirstStartBits); + crossing_info *object_starts; public: // If we're setting first_start, assume the card has an object. inline void set_first_start(size_t card_index, uint8_t value) { - assert(value < FirstStartBits, "Offset into card would be truncated"); object_starts[card_index].offsets.first = ObjectStartsInCardRegion | value; } From 238fb7a7fe9d9c5dfdba807a84e223a4ed372cb2 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Thu, 18 May 2023 20:44:01 +0000 Subject: [PATCH 235/254] Use soft max capacity only for trigger calculations Reviewed-by: kdnilsen, ysr --- .../shenandoahAdaptiveHeuristics.cpp | 9 ++++----- .../gc/shenandoah/shenandoahGeneration.cpp | 20 +++++++++++-------- .../gc/shenandoah/shenandoahGeneration.hpp | 16 ++++++++++----- .../share/gc/shenandoah/shenandoahHeap.cpp | 11 +++++----- test/hotspot/jtreg/ProblemList.txt | 3 +-- 5 files changed, 33 insertions(+), 26 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp index b37bedbcb146a..a6d2dc47f6b68 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp @@ -325,14 +325,13 @@ static double saturate(double value, double min, double max) { } bool ShenandoahAdaptiveHeuristics::should_start_gc() { - size_t max_capacity = _generation->max_capacity(); size_t capacity = _generation->soft_max_capacity(); - size_t available = _generation->available(); + size_t available = _generation->soft_available(); size_t allocated = _generation->bytes_allocated_since_gc_start(); log_debug(gc)("should_start_gc (%s)? available: " SIZE_FORMAT ", soft_max_capacity: " SIZE_FORMAT - ", max_capacity: " SIZE_FORMAT ", allocated: " SIZE_FORMAT, - _generation->name(), available, capacity, max_capacity, allocated); + ", allocated: " SIZE_FORMAT, + _generation->name(), available, capacity, allocated); // The collector reserve may eat into what the mutator is allowed to use. Make sure we are looking // at what is available to the mutator when deciding whether to start a GC. @@ -378,7 +377,7 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { // 1. At certain phase changes, we may discard large amounts of data and replace it with large numbers of newly // allocated objects. This "spike" looks more like a phase change. We were in steady state at M bytes/sec // allocation rate and now we're in a "reinitialization phase" that looks like N bytes/sec. We need the "spike" - // accomodation to give us enough runway to recalibrate our "average allocation rate". + // accommodation to give us enough runway to recalibrate our "average allocation rate". // // 2. The typical workload changes. "Suddenly", our typical workload of N TPS increases to N+delta TPS. This means // our average allocation rate needs to be adjusted. Once again, we need the "spike" accomodation to give us diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index 8755883b2c50f..22281b3250e25 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -262,12 +262,12 @@ void ShenandoahGeneration::compute_evacuation_budgets(ShenandoahHeap* heap, bool old_evacuation_reserve = old_generation->available(); // This assertion has been disabled because we expect this code to be replaced by 05/2023 // assert(old_evacuation_reserve > minimum_evacuation_reserve, "Old-gen available has not been preserved!"); - size_t old_evac_reserve_max = old_generation->soft_max_capacity() * ShenandoahOldEvacReserve / 100; + size_t old_evac_reserve_max = old_generation->max_capacity() * ShenandoahOldEvacReserve / 100; if (old_evac_reserve_max < old_evacuation_reserve) { old_evacuation_reserve = old_evac_reserve_max; } young_evac_reserve_max = - (((young_generation->soft_max_capacity() * ShenandoahEvacReserve) / 100) * ShenandoahOldEvacRatioPercent) / 100; + (((young_generation->max_capacity() * ShenandoahEvacReserve) / 100) * ShenandoahOldEvacRatioPercent) / 100; if (young_evac_reserve_max < old_evacuation_reserve) { old_evacuation_reserve = young_evac_reserve_max; } @@ -897,7 +897,7 @@ ShenandoahGeneration::ShenandoahGeneration(ShenandoahGenerationType type, _collection_thread_time_s(0.0), _affiliated_region_count(0), _humongous_waste(0), _used(0), _bytes_allocated_since_gc_start(0), _max_capacity(max_capacity), _soft_max_capacity(soft_max_capacity), - _adjusted_capacity(soft_max_capacity), _heuristics(nullptr) { + _adjusted_capacity(max_capacity), _heuristics(nullptr) { _is_marking_complete.set(); assert(max_workers > 0, "At least one queue"); for (uint i = 0; i < max_workers; ++i) { @@ -994,7 +994,7 @@ size_t ShenandoahGeneration::used_regions() const { } size_t ShenandoahGeneration::free_unaffiliated_regions() const { - size_t result = soft_max_capacity() / ShenandoahHeapRegion::region_size_bytes(); + size_t result = max_capacity() / ShenandoahHeapRegion::region_size_bytes(); if (_affiliated_region_count > result) { result = 0; // If old-gen is loaning regions to young-gen, affiliated regions may exceed capacity temporarily. } else { @@ -1008,6 +1008,12 @@ size_t ShenandoahGeneration::used_regions_size() const { } size_t ShenandoahGeneration::available() const { + size_t in_use = used() + get_humongous_waste(); + size_t capacity = max_capacity(); + return in_use > capacity ? 0 : capacity - in_use; +} + +size_t ShenandoahGeneration::soft_available() const { size_t in_use = used() + get_humongous_waste(); size_t soft_capacity = soft_max_capacity(); return in_use > soft_capacity ? 0 : soft_capacity - in_use; @@ -1016,12 +1022,12 @@ size_t ShenandoahGeneration::available() const { size_t ShenandoahGeneration::adjust_available(intptr_t adjustment) { assert(adjustment % ShenandoahHeapRegion::region_size_bytes() == 0, "Adjustment to generation size must be multiple of region size"); - _adjusted_capacity = soft_max_capacity() + adjustment; + _adjusted_capacity = max_capacity() + adjustment; return _adjusted_capacity; } size_t ShenandoahGeneration::unadjust_available() { - _adjusted_capacity = soft_max_capacity(); + _adjusted_capacity = max_capacity(); return _adjusted_capacity; } @@ -1054,7 +1060,6 @@ void ShenandoahGeneration::increase_capacity(size_t increment) { increment, ShenandoahHeapRegion::region_size_bytes()); } _max_capacity += increment; - _soft_max_capacity += increment; _adjusted_capacity += increment; } @@ -1068,7 +1073,6 @@ void ShenandoahGeneration::decrease_capacity(size_t decrement) { decrement, ShenandoahHeapRegion::region_size_bytes()); } _max_capacity -= decrement; - _soft_max_capacity -= decrement; _adjusted_capacity -= decrement; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index 9a92c966d9351..160a51a443b98 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -111,21 +111,27 @@ class ShenandoahGeneration : public CHeapObj { virtual size_t used() const { return _used; } virtual size_t available() const; + // Returns the memory available based on the _soft_ max heap capacity (soft_max_heap - used). + // The soft max heap size may be adjusted lower than the max heap size to cause the trigger + // to believe it has less memory available than is _really_ available. Lowering the soft + // max heap size will cause the adaptive heuristic to run more frequent cycles. + size_t soft_available() const; + // During evacuation and update-refs, some memory may be shifted between generations. In particular, memory // may be loaned by old-gen to young-gen based on the promise the loan will be promptly repaid from the memory reclaimed // when the current collection set is recycled. The capacity adjustment also takes into consideration memory that is // set aside within each generation to hold the results of evacuation, but not promotion, into that region. Promotions // into old-gen are bounded by adjusted_available() whereas evacuations into old-gen are pre-committed. - virtual size_t adjusted_available() const; - virtual size_t adjusted_capacity() const; + size_t adjusted_available() const; + size_t adjusted_capacity() const; // This is the number of FREE regions that are eligible to be affiliated with this generation according to the current // adjusted capacity. - virtual size_t adjusted_unaffiliated_regions() const; + size_t adjusted_unaffiliated_regions() const; // Both of following return new value of available - virtual size_t adjust_available(intptr_t adjustment); - virtual size_t unadjust_available(); + size_t adjust_available(intptr_t adjustment); + size_t unadjust_available(); size_t bytes_allocated_since_gc_start(); void reset_bytes_allocated_since_gc_start(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index a99fda65a1fa3..ab2de534b6c85 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -511,7 +511,7 @@ void ShenandoahHeap::initialize_heuristics_generations() { // for old would be total heap - minimum capacity of young. This means the sum of the maximum // allowed for old and young could exceed the total heap size. It remains the case that the // _actual_ capacity of young + old = total. - _generation_sizer.heap_size_changed(soft_max_capacity()); + _generation_sizer.heap_size_changed(max_capacity()); size_t initial_capacity_young = _generation_sizer.max_young_size(); size_t max_capacity_young = _generation_sizer.max_young_size(); size_t initial_capacity_old = max_capacity() - max_capacity_young; @@ -820,11 +820,10 @@ void ShenandoahHeap::set_soft_max_capacity(size_t v) { Atomic::store(&_soft_max_size, v); if (mode()->is_generational()) { - _generation_sizer.heap_size_changed(_soft_max_size); - size_t soft_max_capacity_young = _generation_sizer.max_young_size(); - size_t soft_max_capacity_old = _soft_max_size - soft_max_capacity_young; - _young_generation->set_soft_max_capacity(soft_max_capacity_young); - _old_generation->set_soft_max_capacity(soft_max_capacity_old); + size_t max_capacity_young = _generation_sizer.max_young_size(); + size_t min_capacity_young = _generation_sizer.min_young_size(); + size_t new_capacity_young = clamp(v, min_capacity_young, max_capacity_young); + _young_generation->set_soft_max_capacity(new_capacity_young); } } diff --git a/test/hotspot/jtreg/ProblemList.txt b/test/hotspot/jtreg/ProblemList.txt index feb579ead6a5f..b199a5232d6a7 100644 --- a/test/hotspot/jtreg/ProblemList.txt +++ b/test/hotspot/jtreg/ProblemList.txt @@ -85,9 +85,8 @@ gc/stress/TestStressG1Humongous.java 8286554 windows-x64 gc/shenandoah/oom/TestThreadFailure.java 8306335 generic-all gc/shenandoah/oom/TestClassLoaderLeak.java 8306336 generic-all -gc/shenandoah/TestDynamicSoftMaxHeapSize.java#generational 8306333 generic-all gc/stress/gclocker/TestGCLockerWithShenandoah.java#generational 8306341 generic-all - +gc/TestAllocHumongousFragment.java#generational 8306342 generic-all ############################################################################# From 77885b1085e2bbd027643c77d2e56bada4b24369 Mon Sep 17 00:00:00 2001 From: William Kemper Date: Fri, 19 May 2023 16:02:48 +0000 Subject: [PATCH 236/254] Improve mixed collection logging Reviewed-by: ysr, kdnilsen --- .../heuristics/shenandoahOldHeuristics.cpp | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp index 4ce60002f376b..ad8535ebe336d 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp @@ -29,9 +29,11 @@ #include "gc/shenandoah/shenandoahHeap.hpp" #include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" #include "gc/shenandoah/shenandoahOldGeneration.hpp" -#include "gc/shenandoah/shenandoahYoungGeneration.hpp" #include "utilities/quickSort.hpp" +#define BYTES_FORMAT SIZE_FORMAT "%s" +#define FORMAT_BYTES(b) byte_size_in_proper_unit(b), proper_unit_for_byte_size(b) + uint ShenandoahOldHeuristics::NOT_FOUND = -1U; ShenandoahOldHeuristics::ShenandoahOldHeuristics(ShenandoahOldGeneration* generation, ShenandoahHeuristics* trigger_heuristic) : @@ -124,6 +126,15 @@ bool ShenandoahOldHeuristics::prime_collection_set(ShenandoahCollectionSet* coll if (all_candidates_are_pinned()) { log_info(gc)("All candidate regions " UINT32_FORMAT " are pinned", unprocessed_old_collection_candidates()); _old_generation->transition_to(ShenandoahOldGeneration::WAITING_FOR_FILL); + } else { + log_info(gc)("No regions selected for mixed collection. " + "Old evacuation budget: " BYTES_FORMAT ", Remaining evacuation budget: " BYTES_FORMAT + ", Lost capacity: " BYTES_FORMAT + ", Next candidate: " UINT32_FORMAT ", Last candidate: " UINT32_FORMAT, + FORMAT_BYTES(heap->get_old_evac_reserve()), + FORMAT_BYTES(remaining_old_evacuation_budget), + FORMAT_BYTES(lost_evacuation_capacity), + _next_old_collection_candidate, _last_old_collection_candidate); } } @@ -454,3 +465,5 @@ void ShenandoahOldHeuristics::choose_collection_set_from_regiondata(ShenandoahCo } +#undef BYTES_FORMAT +#undef FORMAT_BYTES From e33ecfaedff193a62ae4dfebc844b94305a569ad Mon Sep 17 00:00:00 2001 From: William Kemper Date: Wed, 24 May 2023 18:52:50 +0000 Subject: [PATCH 237/254] Make generational mode experimental Reviewed-by: kdnilsen --- .../gc/shenandoah/mode/shenandoahGenerationalMode.hpp | 2 +- .../shenandoah/generational/TestCLIModeGenerational.java | 1 + .../shenandoah/generational/TestSimpleGenerational.java | 1 + .../jtreg/gc/shenandoah/options/TestModeUnlock.java | 8 ++++---- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp index 71327bd5189af..64ac2f03374a5 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.hpp @@ -33,7 +33,7 @@ class ShenandoahGenerationalMode : public ShenandoahMode { virtual ShenandoahHeuristics* initialize_heuristics(ShenandoahGeneration* generation) const; virtual const char* name() { return "Generational"; } virtual bool is_diagnostic() { return false; } - virtual bool is_experimental() { return false; } + virtual bool is_experimental() { return true; } virtual bool is_generational() { return true; } }; diff --git a/test/hotspot/jtreg/gc/shenandoah/generational/TestCLIModeGenerational.java b/test/hotspot/jtreg/gc/shenandoah/generational/TestCLIModeGenerational.java index dd9210aaaf21f..183787ced46cf 100644 --- a/test/hotspot/jtreg/gc/shenandoah/generational/TestCLIModeGenerational.java +++ b/test/hotspot/jtreg/gc/shenandoah/generational/TestCLIModeGenerational.java @@ -35,6 +35,7 @@ * @run main/othervm -Xbootclasspath/a:. * -XX:+IgnoreUnrecognizedVMOptions * -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI + * -XX:+UnlockExperimentalVMOptions * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational * gc.shenandoah.generational.TestCLIModeGenerational */ diff --git a/test/hotspot/jtreg/gc/shenandoah/generational/TestSimpleGenerational.java b/test/hotspot/jtreg/gc/shenandoah/generational/TestSimpleGenerational.java index 4a9d3efc7191c..a305d00102f61 100644 --- a/test/hotspot/jtreg/gc/shenandoah/generational/TestSimpleGenerational.java +++ b/test/hotspot/jtreg/gc/shenandoah/generational/TestSimpleGenerational.java @@ -37,6 +37,7 @@ * @run main/othervm -Xbootclasspath/a:. * -XX:+IgnoreUnrecognizedVMOptions * -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI + * -XX:+UnlockExperimentalVMOptions * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational * gc.shenandoah.generational.TestSimpleGenerational */ diff --git a/test/hotspot/jtreg/gc/shenandoah/options/TestModeUnlock.java b/test/hotspot/jtreg/gc/shenandoah/options/TestModeUnlock.java index c0b7690d77846..e8b3798b6419e 100644 --- a/test/hotspot/jtreg/gc/shenandoah/options/TestModeUnlock.java +++ b/test/hotspot/jtreg/gc/shenandoah/options/TestModeUnlock.java @@ -45,10 +45,10 @@ enum Mode { } public static void main(String[] args) throws Exception { - testWith("-XX:ShenandoahGCMode=satb", Mode.PRODUCT); - testWith("-XX:ShenandoahGCMode=iu", Mode.EXPERIMENTAL); - testWith("-XX:ShenandoahGCMode=passive", Mode.DIAGNOSTIC); - testWith("-XX:ShenandoahGCMode=generational", Mode.PRODUCT); + testWith("-XX:ShenandoahGCMode=satb", Mode.PRODUCT); + testWith("-XX:ShenandoahGCMode=iu", Mode.EXPERIMENTAL); + testWith("-XX:ShenandoahGCMode=passive", Mode.DIAGNOSTIC); + testWith("-XX:ShenandoahGCMode=generational", Mode.EXPERIMENTAL); } private static void testWith(String h, Mode mode) throws Exception { From 57fdc1865d89b0f82d8601a813b9e9eb2d8abec2 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Fri, 26 May 2023 18:40:12 +0000 Subject: [PATCH 238/254] Expand old on demand Reviewed-by: ysr --- .../shenandoahAdaptiveHeuristics.cpp | 419 ++++++---- .../shenandoahAdaptiveHeuristics.hpp | 16 +- .../shenandoahCompactHeuristics.cpp | 3 +- .../heuristics/shenandoahHeuristics.cpp | 251 +++++- .../heuristics/shenandoahHeuristics.hpp | 17 +- .../heuristics/shenandoahOldHeuristics.cpp | 252 ++++-- .../heuristics/shenandoahOldHeuristics.hpp | 32 +- .../gc/shenandoah/shenandoahCollectionSet.cpp | 11 +- .../gc/shenandoah/shenandoahCollectionSet.hpp | 9 + .../shenandoah/shenandoahCollectorPolicy.cpp | 6 + .../shenandoah/shenandoahCollectorPolicy.hpp | 6 +- .../gc/shenandoah/shenandoahConcurrentGC.cpp | 174 +++-- .../gc/shenandoah/shenandoahControlThread.cpp | 92 ++- .../gc/shenandoah/shenandoahControlThread.hpp | 7 +- .../gc/shenandoah/shenandoahDegeneratedGC.cpp | 65 +- .../share/gc/shenandoah/shenandoahFreeSet.cpp | 271 +++++-- .../share/gc/shenandoah/shenandoahFreeSet.hpp | 19 +- .../share/gc/shenandoah/shenandoahFullGC.cpp | 107 ++- .../gc/shenandoah/shenandoahGeneration.cpp | 735 +++++------------- .../gc/shenandoah/shenandoahGeneration.hpp | 30 +- .../share/gc/shenandoah/shenandoahHeap.cpp | 458 ++++++++--- .../share/gc/shenandoah/shenandoahHeap.hpp | 91 ++- .../gc/shenandoah/shenandoahHeap.inline.hpp | 23 +- .../gc/shenandoah/shenandoahHeapRegion.cpp | 199 +++-- .../gc/shenandoah/shenandoahHeapRegion.hpp | 13 +- .../shenandoahHeapRegion.inline.hpp | 25 + .../gc/shenandoah/shenandoahInitLogger.cpp | 4 +- .../gc/shenandoah/shenandoahMmuTracker.cpp | 292 +++---- .../gc/shenandoah/shenandoahMmuTracker.hpp | 75 +- .../share/gc/shenandoah/shenandoahOldGC.cpp | 44 ++ .../gc/shenandoah/shenandoahOldGeneration.cpp | 30 +- .../gc/shenandoah/shenandoahOldGeneration.hpp | 32 +- .../shenandoah/shenandoahRegulatorThread.cpp | 3 +- .../gc/shenandoah/shenandoahVerifier.cpp | 98 ++- .../gc/shenandoah/shenandoahVerifier.hpp | 12 + .../gc/shenandoah/shenandoah_globals.hpp | 86 +- test/hotspot/jtreg/ProblemList.txt | 5 - 37 files changed, 2533 insertions(+), 1479 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp index a6d2dc47f6b68..aba8f693cf8e1 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.cpp @@ -30,6 +30,8 @@ #include "gc/shenandoah/shenandoahFreeSet.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahGeneration.hpp" +#include "gc/shenandoah/shenandoahHeapRegion.inline.hpp" +#include "gc/shenandoah/shenandoahOldGeneration.hpp" #include "gc/shenandoah/shenandoahYoungGeneration.hpp" #include "logging/log.hpp" #include "logging/logTag.hpp" @@ -56,9 +58,6 @@ const double ShenandoahAdaptiveHeuristics::HIGHEST_EXPECTED_AVAILABLE_AT_END = 0 const double ShenandoahAdaptiveHeuristics::MINIMUM_CONFIDENCE = 0.319; // 25% const double ShenandoahAdaptiveHeuristics::MAXIMUM_CONFIDENCE = 3.291; // 99.9% -// TODO: Provide comment here or remove if not used -const uint ShenandoahAdaptiveHeuristics::MINIMUM_RESIZE_INTERVAL = 10; - ShenandoahAdaptiveHeuristics::ShenandoahAdaptiveHeuristics(ShenandoahGeneration* generation) : ShenandoahHeuristics(generation), _margin_of_error_sd(ShenandoahAdaptiveInitialConfidence), @@ -110,10 +109,27 @@ void ShenandoahAdaptiveHeuristics::choose_collection_set_from_regiondata(Shenand QuickSort::sort(data, (int)size, compare_by_garbage, false); if (is_generational) { + for (size_t idx = 0; idx < size; idx++) { + ShenandoahHeapRegion* r = data[idx]._region; + if (cset->is_preselected(r->index())) { + assert(r->age() >= InitialTenuringThreshold, "Preselected regions must have tenure age"); + // Entire region will be promoted, This region does not impact young-gen or old-gen evacuation reserve. + // This region has been pre-selected and its impact on promotion reserve is already accounted for. + + // r->used() is r->garbage() + r->get_live_data_bytes() + // Since all live data in this region is being evacuated from young-gen, it is as if this memory + // is garbage insofar as young-gen is concerned. Counting this as garbage reduces the need to + // reclaim highly utilized young-gen regions just for the sake of finding min_garbage to reclaim + // within youn-gen memory. + + cur_young_garbage += r->garbage(); + cset->add_region(r); + } + } if (is_global) { size_t max_young_cset = (size_t) (heap->get_young_evac_reserve() / ShenandoahEvacWaste); size_t young_cur_cset = 0; - size_t max_old_cset = (size_t) (heap->get_old_evac_reserve() / ShenandoahEvacWaste); + size_t max_old_cset = (size_t) (heap->get_old_evac_reserve() / ShenandoahOldEvacWaste); size_t old_cur_cset = 0; size_t free_target = (capacity * ShenandoahMinFreeThreshold) / 100 + max_young_cset; size_t min_garbage = (free_target > actual_free) ? (free_target - actual_free) : 0; @@ -126,6 +142,9 @@ void ShenandoahAdaptiveHeuristics::choose_collection_set_from_regiondata(Shenand for (size_t idx = 0; idx < size; idx++) { ShenandoahHeapRegion* r = data[idx]._region; + if (cset->is_preselected(r->index())) { + continue; + } bool add_region = false; if (r->is_old()) { size_t new_cset = old_cur_cset + r->get_live_data_bytes(); @@ -133,17 +152,6 @@ void ShenandoahAdaptiveHeuristics::choose_collection_set_from_regiondata(Shenand add_region = true; old_cur_cset = new_cset; } - } else if (cset->is_preselected(r->index())) { - assert(r->age() >= InitialTenuringThreshold, "Preselected regions must have tenure age"); - // Entire region will be promoted, This region does not impact young-gen or old-gen evacuation reserve. - // This region has been pre-selected and its impact on promotion reserve is already accounted for. - add_region = true; - // r->used() is r->garbage() + r->get_live_data_bytes() - // Since all live data in this region is being evacuated from young-gen, it is as if this memory - // is garbage insofar as young-gen is concerned. Counting this as garbage reduces the need to - // reclaim highly utilized young-gen regions just for the sake of finding min_garbage to reclaim - // within youn-gen memory. - cur_young_garbage += r->used(); } else if (r->age() < InitialTenuringThreshold) { size_t new_cset = young_cur_cset + r->get_live_data_bytes(); size_t region_garbage = r->garbage(); @@ -176,42 +184,29 @@ void ShenandoahAdaptiveHeuristics::choose_collection_set_from_regiondata(Shenand for (size_t idx = 0; idx < size; idx++) { ShenandoahHeapRegion* r = data[idx]._region; - bool add_region = false; - - if (!r->is_old()) { - if (cset->is_preselected(r->index())) { - assert(r->age() >= InitialTenuringThreshold, "Preselected regions must have tenure age"); - // Entire region will be promoted, This region does not impact young-gen evacuation reserve. Memory has already - // been set aside to hold evacuation results as advance_promotion_reserve. - add_region = true; - // Since all live data in this region is being evacuated from young-gen, it is as if this memory - // is garbage insofar as young-gen is concerned. Counting this as garbage reduces the need to - // reclaim highly utilized young-gen regions just for the sake of finding min_garbage to reclaim - // within youn-gen memory - cur_young_garbage += r->get_live_data_bytes(); - } else if (r->age() < InitialTenuringThreshold) { - size_t new_cset = cur_cset + r->get_live_data_bytes(); - size_t region_garbage = r->garbage(); - size_t new_garbage = cur_young_garbage + region_garbage; - bool add_regardless = (region_garbage > ignore_threshold) && (new_garbage < min_garbage); - if ((new_cset <= max_cset) && (add_regardless || (region_garbage > garbage_threshold))) { - add_region = true; - cur_cset = new_cset; - cur_young_garbage = new_garbage; - } - } - // Note that we do not add aged regions if they were not pre-selected. The reason they were not preselected - // is because there is not sufficient room in old-gen to hold their to-be-promoted live objects. - - if (add_region) { + if (cset->is_preselected(r->index())) { + continue; + } + if (r->age() < InitialTenuringThreshold) { + size_t new_cset = cur_cset + r->get_live_data_bytes(); + size_t region_garbage = r->garbage(); + size_t new_garbage = cur_young_garbage + region_garbage; + bool add_regardless = (region_garbage > ignore_threshold) && (new_garbage < min_garbage); + assert(r->is_young(), "Only young candidates expected in the data array"); + if ((new_cset <= max_cset) && (add_regardless || (region_garbage > garbage_threshold))) { + cur_cset = new_cset; + cur_young_garbage = new_garbage; cset->add_region(r); } } + // Note that we do not add aged regions if they were not pre-selected. The reason they were not preselected + // is because there is not sufficient room in old-gen to hold their to-be-promoted live objects or because + // they are to be promoted in place. } } } else { // Traditional Shenandoah (non-generational) - size_t capacity = ShenandoahHeap::heap()->soft_max_capacity(); + size_t capacity = ShenandoahHeap::heap()->max_capacity(); size_t max_cset = (size_t)((1.0 * capacity / 100 * ShenandoahEvacReserve) / ShenandoahEvacWaste); size_t free_target = (capacity * ShenandoahMinFreeThreshold) / 100 + max_cset; size_t min_garbage = (free_target > actual_free) ? (free_target - actual_free) : 0; @@ -243,12 +238,21 @@ void ShenandoahAdaptiveHeuristics::choose_collection_set_from_regiondata(Shenand } } } + + size_t collected_old = cset->get_old_bytes_reserved_for_evacuation(); + size_t collected_promoted = cset->get_young_bytes_to_be_promoted(); + size_t collected_young = cset->get_young_bytes_reserved_for_evacuation(); + + log_info(gc, ergo)("Chosen CSet evacuates young: " SIZE_FORMAT "%s (of which at least: " SIZE_FORMAT "%s are to be promoted), " + "old: " SIZE_FORMAT "%s", + byte_size_in_proper_unit(collected_young), proper_unit_for_byte_size(collected_young), + byte_size_in_proper_unit(collected_promoted), proper_unit_for_byte_size(collected_promoted), + byte_size_in_proper_unit(collected_old), proper_unit_for_byte_size(collected_old)); } void ShenandoahAdaptiveHeuristics::record_cycle_start() { ShenandoahHeuristics::record_cycle_start(); _allocation_rate.allocation_counter_reset(); - ++_cycles_since_last_resize; } void ShenandoahAdaptiveHeuristics::record_success_concurrent(bool abbreviated) { @@ -324,6 +328,84 @@ static double saturate(double value, double min, double max) { return MAX2(MIN2(value, max), min); } +// Return a conservative estimate of how much memory can be allocated before we need to start GC. The estimate is based +// on memory that is currently available within young generation plus all of the memory that will be added to the young +// generation at the end of the current cycle (as represented by young_regions_to_be_reclaimed) and on the anticipated +// amount of time required to perform a GC. +size_t ShenandoahAdaptiveHeuristics::bytes_of_allocation_runway_before_gc_trigger(size_t young_regions_to_be_reclaimed) { + assert(_generation->is_young(), "Only meaningful for young-gen heuristic"); + + size_t max_capacity = _generation->max_capacity(); + size_t capacity = _generation->soft_max_capacity(); + size_t usage = _generation->used(); + size_t available = (capacity > usage)? capacity - usage: 0; + size_t allocated = _generation->bytes_allocated_since_gc_start(); + + size_t available_young_collected = ShenandoahHeap::heap()->collection_set()->get_young_available_bytes_collected(); + size_t anticipated_available = + available + young_regions_to_be_reclaimed * ShenandoahHeapRegion::region_size_bytes() - available_young_collected; + size_t allocation_headroom = anticipated_available; + size_t spike_headroom = capacity * ShenandoahAllocSpikeFactor / 100; + size_t penalties = capacity * _gc_time_penalties / 100; + + double rate = _allocation_rate.sample(allocated); + + // At what value of available, would avg and spike triggers occur? + // if allocation_headroom < avg_cycle_time * avg_alloc_rate, then we experience avg trigger + // if allocation_headroom < avg_cycle_time * rate, then we experience spike trigger if is_spiking + // + // allocation_headroom = + // 0, if penalties > available or if penalties + spike_headroom > available + // available - penalties - spike_headroom, otherwise + // + // so we trigger if available - penalties - spike_headroom < avg_cycle_time * avg_alloc_rate, which is to say + // available < avg_cycle_time * avg_alloc_rate + penalties + spike_headroom + // or if available < penalties + spike_headroom + // + // since avg_cycle_time * avg_alloc_rate > 0, the first test is sufficient to test both conditions + // + // thus, evac_slack_avg is MIN2(0, available - avg_cycle_time * avg_alloc_rate + penalties + spike_headroom) + // + // similarly, evac_slack_spiking is MIN2(0, available - avg_cycle_time * rate + penalties + spike_headroom) + // but evac_slack_spiking is only relevant if is_spiking, as defined below. + + double avg_cycle_time = _gc_cycle_time_history->davg() + (_margin_of_error_sd * _gc_cycle_time_history->dsd()); + + // TODO: Consider making conservative adjustments to avg_cycle_time, such as: (avg_cycle_time *= 2) in cases where + // we expect a longer-than-normal GC duration. This includes mixed evacuations, evacuation that perform promotion + // including promotion in place, and OLD GC bootstrap cycles. It has been observed that these cycles sometimes + // require twice or more the duration of "normal" GC cycles. We have experimented with this approach. While it + // does appear to reduce the frequency of degenerated cycles due to late triggers, it also has the effect of reducing + // evacuation slack so that there is less memory available to be transferred to OLD. The result is that we + // throttle promotion and it takes too long to move old objects out of the young generation. + + double avg_alloc_rate = _allocation_rate.upper_bound(_margin_of_error_sd); + size_t evac_slack_avg; + if (anticipated_available > avg_cycle_time * avg_alloc_rate + penalties + spike_headroom) { + evac_slack_avg = anticipated_available - (avg_cycle_time * avg_alloc_rate + penalties + spike_headroom); + } else { + // we have no slack because it's already time to trigger + evac_slack_avg = 0; + } + + bool is_spiking = _allocation_rate.is_spiking(rate, _spike_threshold_sd); + size_t evac_slack_spiking; + if (is_spiking) { + if (anticipated_available > avg_cycle_time * rate + penalties + spike_headroom) { + evac_slack_spiking = anticipated_available - (avg_cycle_time * rate + penalties + spike_headroom); + } else { + // we have no slack because it's already time to trigger + evac_slack_spiking = 0; + } + } else { + evac_slack_spiking = evac_slack_avg; + } + + size_t threshold = min_free_threshold(); + size_t evac_min_threshold = (anticipated_available > threshold)? anticipated_available - threshold: 0; + return MIN3(evac_slack_spiking, evac_slack_avg, evac_min_threshold); +} + bool ShenandoahAdaptiveHeuristics::should_start_gc() { size_t capacity = _generation->soft_max_capacity(); size_t available = _generation->soft_available(); @@ -347,137 +429,144 @@ bool ShenandoahAdaptiveHeuristics::should_start_gc() { double rate = _allocation_rate.sample(allocated); _last_trigger = OTHER; - size_t min_threshold = min_free_threshold(); - - if (available < min_threshold) { - log_info(gc)("Trigger (%s): Free (" SIZE_FORMAT "%s) is below minimum threshold (" SIZE_FORMAT "%s)", - _generation->name(), - byte_size_in_proper_unit(available), proper_unit_for_byte_size(available), - byte_size_in_proper_unit(min_threshold), proper_unit_for_byte_size(min_threshold)); - return resize_and_evaluate(); - } - - // Check if we need to learn a bit about the application - const size_t max_learn = ShenandoahLearningSteps; - if (_gc_times_learned < max_learn) { - size_t init_threshold = capacity / 100 * ShenandoahInitFreeThreshold; - if (available < init_threshold) { - log_info(gc)("Trigger (%s): Learning " SIZE_FORMAT " of " SIZE_FORMAT ". Free (" SIZE_FORMAT "%s) is below initial threshold (" SIZE_FORMAT "%s)", - _generation->name(), _gc_times_learned + 1, max_learn, - byte_size_in_proper_unit(available), proper_unit_for_byte_size(available), - byte_size_in_proper_unit(init_threshold), proper_unit_for_byte_size(init_threshold)); + // OLD generation is maintained to be as small as possible. Depletion-of-free-pool triggers do not apply to old generation. + if (!_generation->is_old()) { + size_t min_threshold = min_free_threshold(); + if (available < min_threshold) { + log_info(gc)("Trigger (%s): Free (" SIZE_FORMAT "%s) is below minimum threshold (" SIZE_FORMAT "%s)", + _generation->name(), + byte_size_in_proper_unit(available), proper_unit_for_byte_size(available), + byte_size_in_proper_unit(min_threshold), proper_unit_for_byte_size(min_threshold)); return true; } - } - // Rationale: - // The idea is that there is an average allocation rate and there are occasional abnormal bursts (or spikes) of - // allocations that exceed the average allocation rate. What do these spikes look like? - // - // 1. At certain phase changes, we may discard large amounts of data and replace it with large numbers of newly - // allocated objects. This "spike" looks more like a phase change. We were in steady state at M bytes/sec - // allocation rate and now we're in a "reinitialization phase" that looks like N bytes/sec. We need the "spike" - // accommodation to give us enough runway to recalibrate our "average allocation rate". - // - // 2. The typical workload changes. "Suddenly", our typical workload of N TPS increases to N+delta TPS. This means - // our average allocation rate needs to be adjusted. Once again, we need the "spike" accomodation to give us - // enough runway to recalibrate our "average allocation rate". - // - // 3. Though there is an "average" allocation rate, a given workload's demand for allocation may be very bursty. We - // allocate a bunch of LABs during the 5 ms that follow completion of a GC, then we perform no more allocations for - // the next 150 ms. It seems we want the "spike" to represent the maximum divergence from average within the - // period of time between consecutive evaluation of the should_start_gc() service. Here's the thinking: - // - // a) Between now and the next time I ask whether should_start_gc(), we might experience a spike representing - // the anticipated burst of allocations. If that would put us over budget, then we should start GC immediately. - // b) Between now and the anticipated depletion of allocation pool, there may be two or more bursts of allocations. - // If there are more than one of these bursts, we can "approximate" that these will be separated by spans of - // time with very little or no allocations so the "average" allocation rate should be a suitable approximation - // of how this will behave. - // - // For cases 1 and 2, we need to "quickly" recalibrate the average allocation rate whenever we detect a change - // in operation mode. We want some way to decide that the average rate has changed. Make average allocation rate - // computations an independent effort. - - - // TODO: Account for inherent delays in responding to GC triggers - // 1. It has been observed that delays of 200 ms or greater are common between the moment we return true from should_start_gc() - // and the moment at which we begin execution of the concurrent reset phase. Add this time into the calculation of - // avg_cycle_time below. (What is "this time"? Perhaps we should remember recent history of this delay for the - // running workload and use the maximum delay recently seen for "this time".) - // 2. The frequency of inquiries to should_start_gc() is adaptive, ranging between ShenandoahControlIntervalMin and - // ShenandoahControlIntervalMax. The current control interval (or the max control interval) should also be added into - // the calculation of avg_cycle_time below. - - // Check if allocation headroom is still okay. This also factors in: - // 1. Some space to absorb allocation spikes (ShenandoahAllocSpikeFactor) - // 2. Accumulated penalties from Degenerated and Full GC - size_t allocation_headroom = available; - size_t spike_headroom = capacity / 100 * ShenandoahAllocSpikeFactor; - size_t penalties = capacity / 100 * _gc_time_penalties; - - allocation_headroom -= MIN2(allocation_headroom, penalties); - allocation_headroom -= MIN2(allocation_headroom, spike_headroom); + // Check if we need to learn a bit about the application + const size_t max_learn = ShenandoahLearningSteps; + if (_gc_times_learned < max_learn) { + size_t init_threshold = capacity / 100 * ShenandoahInitFreeThreshold; + if (available < init_threshold) { + log_info(gc)("Trigger (%s): Learning " SIZE_FORMAT " of " SIZE_FORMAT ". Free (" + SIZE_FORMAT "%s) is below initial threshold (" SIZE_FORMAT "%s)", + _generation->name(), _gc_times_learned + 1, max_learn, + byte_size_in_proper_unit(available), proper_unit_for_byte_size(available), + byte_size_in_proper_unit(init_threshold), proper_unit_for_byte_size(init_threshold)); + return true; + } + } - double avg_cycle_time = _gc_cycle_time_history->davg() + (_margin_of_error_sd * _gc_cycle_time_history->dsd()); + // Rationale: + // The idea is that there is an average allocation rate and there are occasional abnormal bursts (or spikes) of + // allocations that exceed the average allocation rate. What do these spikes look like? + // + // 1. At certain phase changes, we may discard large amounts of data and replace it with large numbers of newly + // allocated objects. This "spike" looks more like a phase change. We were in steady state at M bytes/sec + // allocation rate and now we're in a "reinitialization phase" that looks like N bytes/sec. We need the "spike" + // accomodation to give us enough runway to recalibrate our "average allocation rate". + // + // 2. The typical workload changes. "Suddenly", our typical workload of N TPS increases to N+delta TPS. This means + // our average allocation rate needs to be adjusted. Once again, we need the "spike" accomodation to give us + // enough runway to recalibrate our "average allocation rate". + // + // 3. Though there is an "average" allocation rate, a given workload's demand for allocation may be very bursty. We + // allocate a bunch of LABs during the 5 ms that follow completion of a GC, then we perform no more allocations for + // the next 150 ms. It seems we want the "spike" to represent the maximum divergence from average within the + // period of time between consecutive evaluation of the should_start_gc() service. Here's the thinking: + // + // a) Between now and the next time I ask whether should_start_gc(), we might experience a spike representing + // the anticipated burst of allocations. If that would put us over budget, then we should start GC immediately. + // b) Between now and the anticipated depletion of allocation pool, there may be two or more bursts of allocations. + // If there are more than one of these bursts, we can "approximate" that these will be separated by spans of + // time with very little or no allocations so the "average" allocation rate should be a suitable approximation + // of how this will behave. + // + // For cases 1 and 2, we need to "quickly" recalibrate the average allocation rate whenever we detect a change + // in operation mode. We want some way to decide that the average rate has changed. Make average allocation rate + // computations an independent effort. - double avg_alloc_rate = _allocation_rate.upper_bound(_margin_of_error_sd); - log_debug(gc)("%s: average GC time: %.2f ms, allocation rate: %.0f %s/s", - _generation->name(), avg_cycle_time * 1000, - byte_size_in_proper_unit(avg_alloc_rate), proper_unit_for_byte_size(avg_alloc_rate)); - - if (avg_cycle_time > allocation_headroom / avg_alloc_rate) { - log_info(gc)("Trigger (%s): Average GC time (%.2f ms) is above the time for average allocation rate (%.0f %sB/s) to deplete free headroom (" SIZE_FORMAT "%s) (margin of error = %.2f)", - _generation->name(), avg_cycle_time * 1000, - byte_size_in_proper_unit(avg_alloc_rate), proper_unit_for_byte_size(avg_alloc_rate), - byte_size_in_proper_unit(allocation_headroom), proper_unit_for_byte_size(allocation_headroom), - _margin_of_error_sd); - - log_info(gc, ergo)("Free headroom: " SIZE_FORMAT "%s (free) - " SIZE_FORMAT "%s (spike) - " SIZE_FORMAT "%s (penalties) = " SIZE_FORMAT "%s", - byte_size_in_proper_unit(available), proper_unit_for_byte_size(available), - byte_size_in_proper_unit(spike_headroom), proper_unit_for_byte_size(spike_headroom), - byte_size_in_proper_unit(penalties), proper_unit_for_byte_size(penalties), - byte_size_in_proper_unit(allocation_headroom), proper_unit_for_byte_size(allocation_headroom)); - - _last_trigger = RATE; - return resize_and_evaluate(); - } - bool is_spiking = _allocation_rate.is_spiking(rate, _spike_threshold_sd); - if (is_spiking && avg_cycle_time > allocation_headroom / rate) { - log_info(gc)("Trigger (%s): Average GC time (%.2f ms) is above the time for instantaneous allocation rate (%.0f %sB/s) to deplete free headroom (" SIZE_FORMAT "%s) (spike threshold = %.2f)", - _generation->name(), avg_cycle_time * 1000, - byte_size_in_proper_unit(rate), proper_unit_for_byte_size(rate), - byte_size_in_proper_unit(allocation_headroom), proper_unit_for_byte_size(allocation_headroom), - _spike_threshold_sd); - _last_trigger = SPIKE; - return resize_and_evaluate(); - } + // Check if allocation headroom is still okay. This also factors in: + // 1. Some space to absorb allocation spikes (ShenandoahAllocSpikeFactor) + // 2. Accumulated penalties from Degenerated and Full GC - return ShenandoahHeuristics::should_start_gc(); -} + size_t allocation_headroom = available; + size_t spike_headroom = capacity / 100 * ShenandoahAllocSpikeFactor; + size_t penalties = capacity / 100 * _gc_time_penalties; -bool ShenandoahAdaptiveHeuristics::resize_and_evaluate() { - ShenandoahHeap* heap = ShenandoahHeap::heap(); - if (!heap->mode()->is_generational()) { - // We only attempt to resize the generations in generational mode. - return true; - } + allocation_headroom -= MIN2(allocation_headroom, penalties); + allocation_headroom -= MIN2(allocation_headroom, spike_headroom); - if (_cycles_since_last_resize <= MINIMUM_RESIZE_INTERVAL) { - log_info(gc, ergo)("Not resizing %s for another " UINT32_FORMAT " cycles", - _generation->name(), _cycles_since_last_resize); - return true; - } + double avg_cycle_time = _gc_cycle_time_history->davg() + (_margin_of_error_sd * _gc_cycle_time_history->dsd()); + double avg_alloc_rate = _allocation_rate.upper_bound(_margin_of_error_sd); + log_debug(gc)("%s: average GC time: %.2f ms, allocation rate: %.0f %s/s", + _generation->name(), + avg_cycle_time * 1000, byte_size_in_proper_unit(avg_alloc_rate), proper_unit_for_byte_size(avg_alloc_rate)); - if (!heap->generation_sizer()->transfer_capacity(_generation)) { - // We could not enlarge our generation, so we must start a gc cycle. - log_info(gc, ergo)("Could not increase size of %s, begin gc cycle", _generation->name()); - return true; - } + if (avg_cycle_time > allocation_headroom / avg_alloc_rate) { + + log_info(gc)("Trigger (%s): Average GC time (%.2f ms) is above the time for average allocation rate (%.0f %sB/s)" + " to deplete free headroom (" SIZE_FORMAT "%s) (margin of error = %.2f)", + _generation->name(), avg_cycle_time * 1000, + byte_size_in_proper_unit(avg_alloc_rate), proper_unit_for_byte_size(avg_alloc_rate), + byte_size_in_proper_unit(allocation_headroom), proper_unit_for_byte_size(allocation_headroom), + _margin_of_error_sd); + + log_info(gc, ergo)("Free headroom: " SIZE_FORMAT "%s (free) - " SIZE_FORMAT "%s (spike) - " + SIZE_FORMAT "%s (penalties) = " SIZE_FORMAT "%s", + byte_size_in_proper_unit(available), proper_unit_for_byte_size(available), + byte_size_in_proper_unit(spike_headroom), proper_unit_for_byte_size(spike_headroom), + byte_size_in_proper_unit(penalties), proper_unit_for_byte_size(penalties), + byte_size_in_proper_unit(allocation_headroom), proper_unit_for_byte_size(allocation_headroom)); - log_info(gc)("Increased size of %s generation, re-evaluate trigger criteria", _generation->name()); - return should_start_gc(); + _last_trigger = RATE; + return true; + } + + bool is_spiking = _allocation_rate.is_spiking(rate, _spike_threshold_sd); + if (is_spiking && avg_cycle_time > allocation_headroom / rate) { + log_info(gc)("Trigger (%s): Average GC time (%.2f ms) is above the time for instantaneous allocation rate (%.0f %sB/s)" + " to deplete free headroom (" SIZE_FORMAT "%s) (spike threshold = %.2f)", + _generation->name(), avg_cycle_time * 1000, + byte_size_in_proper_unit(rate), proper_unit_for_byte_size(rate), + byte_size_in_proper_unit(allocation_headroom), proper_unit_for_byte_size(allocation_headroom), + _spike_threshold_sd); + _last_trigger = SPIKE; + return true; + } + + ShenandoahHeap* heap = ShenandoahHeap::heap(); + if (heap->mode()->is_generational()) { + // Get through promotions and mixed evacuations as quickly as possible. These cycles sometimes require significantly + // more time than traditional young-generation cycles so start them up as soon as possible. This is a "mitigation" + // for the reality that old-gen and young-gen activities are not truly "concurrent". If there is old-gen work to + // be done, we start up the young-gen GC threads so they can do some of this old-gen work. As implemented, promotion + // gets priority over old-gen marking. + + size_t promo_potential = heap->get_promotion_potential(); + size_t promo_in_place_potential = heap->get_promotion_in_place_potential(); + ShenandoahOldHeuristics* old_heuristics = (ShenandoahOldHeuristics*) heap->old_generation()->heuristics(); + size_t mixed_candidates = old_heuristics->unprocessed_old_collection_candidates(); + if (promo_potential > 0) { + // Detect unsigned arithmetic underflow + assert(promo_potential < heap->capacity(), "Sanity"); + log_info(gc)("Trigger (%s): expedite promotion of " SIZE_FORMAT "%s", + _generation->name(), byte_size_in_proper_unit(promo_potential), proper_unit_for_byte_size(promo_potential)); + return true; + } else if (promo_in_place_potential > 0) { + // Detect unsigned arithmetic underflow + assert(promo_in_place_potential < heap->capacity(), "Sanity"); + log_info(gc)("Trigger (%s): expedite promotion in place of " SIZE_FORMAT "%s", _generation->name(), + byte_size_in_proper_unit(promo_in_place_potential), + proper_unit_for_byte_size(promo_in_place_potential)); + return true; + } else if (mixed_candidates > 0) { + // We need to run young GC in order to open up some free heap regions so we can finish mixed evacuations. + log_info(gc)("Trigger (%s): expedite mixed evacuation of " SIZE_FORMAT " regions", + _generation->name(), mixed_candidates); + return true; + } + } + } + return ShenandoahHeuristics::should_start_gc(); } void ShenandoahAdaptiveHeuristics::adjust_last_trigger_parameters(double amount) { diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp index 85b563f146d20..494173751eaf0 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahAdaptiveHeuristics.hpp @@ -39,7 +39,6 @@ class ShenandoahAllocationRate : public CHeapObj { double upper_bound(double sds) const; bool is_spiking(double rate, double threshold) const; - private: double instantaneous_rate(double time, size_t allocated) const; @@ -72,6 +71,8 @@ class ShenandoahAdaptiveHeuristics : public ShenandoahHeuristics { virtual bool is_diagnostic() { return false; } virtual bool is_experimental() { return false; } + virtual size_t bytes_of_allocation_runway_before_gc_trigger(size_t young_regions_to_be_recycled); + private: // These are used to adjust the margin of error and the spike threshold // in response to GC cycle outcomes. These values are shared, but the @@ -85,13 +86,6 @@ class ShenandoahAdaptiveHeuristics : public ShenandoahHeuristics { const static double LOWEST_EXPECTED_AVAILABLE_AT_END; const static double HIGHEST_EXPECTED_AVAILABLE_AT_END; - // At least this many cycles must execute before the heuristic will attempt - // to resize its generation. This is to prevent the heuristic from rapidly - // maxing out the generation size (which only forces the collector for the - // other generation to run more frequently, defeating the purpose of improving - // MMU). - const static uint MINIMUM_RESIZE_INTERVAL; - friend class ShenandoahAllocationRate; // Used to record the last trigger that signaled to start a GC. @@ -106,8 +100,6 @@ class ShenandoahAdaptiveHeuristics : public ShenandoahHeuristics { void adjust_margin_of_error(double amount); void adjust_spike_threshold(double amount); - bool resize_and_evaluate(); - ShenandoahAllocationRate _allocation_rate; // The margin of error expressed in standard deviations to add to our @@ -135,10 +127,6 @@ class ShenandoahAdaptiveHeuristics : public ShenandoahHeuristics { // establishes what is 'normal' for the application and is used as a // source of feedback to adjust trigger parameters. TruncatedSeq _available; - - // Do not attempt to resize the generation for this heuristic until this - // value is greater than MINIMUM_RESIZE_INTERVAL. - uint _cycles_since_last_resize; }; #endif // SHARE_GC_SHENANDOAH_HEURISTICS_SHENANDOAHADAPTIVEHEURISTICS_HPP diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp index a84c23b678cd5..250d131730603 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp @@ -49,7 +49,8 @@ ShenandoahCompactHeuristics::ShenandoahCompactHeuristics(ShenandoahGeneration* g bool ShenandoahCompactHeuristics::should_start_gc() { size_t max_capacity = _generation->max_capacity(); size_t capacity = _generation->soft_max_capacity(); - size_t available = _generation->available(); + size_t usage = _generation->used(); + size_t available = (capacity > usage)? capacity - usage: 0; // Make sure the code below treats available without the soft tail. size_t soft_tail = max_capacity - capacity; diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp index ec0c365b873b1..93a4f41432511 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.cpp @@ -39,11 +39,22 @@ #include "logging/log.hpp" #include "logging/logTag.hpp" #include "runtime/globals_extension.hpp" +#include "utilities/quickSort.hpp" +// sort by decreasing garbage (so most garbage comes first) int ShenandoahHeuristics::compare_by_garbage(RegionData a, RegionData b) { - if (a._garbage > b._garbage) + if (a._u._garbage > b._u._garbage) return -1; - else if (a._garbage < b._garbage) + else if (a._u._garbage < b._u._garbage) + return 1; + else return 0; +} + +// sort by increasing live (so least live comes first) +int ShenandoahHeuristics::compare_by_live(RegionData a, RegionData b) { + if (a._u._live_data < b._u._live_data) + return -1; + else if (a._u._live_data > b._u._live_data) return 1; else return 0; } @@ -76,29 +87,170 @@ ShenandoahHeuristics::~ShenandoahHeuristics() { FREE_C_HEAP_ARRAY(RegionGarbage, _region_data); } -size_t ShenandoahHeuristics::select_aged_regions(size_t old_available, size_t num_regions, bool* preselected_regions) { +typedef struct { + ShenandoahHeapRegion* _region; + size_t _live_data; +} AgedRegionData; + +static int compare_by_aged_live(AgedRegionData a, AgedRegionData b) { + if (a._live_data < b._live_data) + return -1; + else if (a._live_data > b._live_data) + return 1; + else return 0; +} + +// Preselect for inclusion into the collection set regions whose age is at or above tenure age which contain more than +// ShenandoahOldGarbageThreshold amounts of garbage. We identify these regions by setting the appropriate entry of +// candidate_regions_for_promotion_by_copy[] to true. All entries are initialized to false before calling this +// function. +// +// During the subsequent selection of the collection set, we give priority to these promotion set candidates. +// Without this prioritization, we found that the aged regions tend to be ignored because they typically have +// much less garbage and much more live data than the recently allocated "eden" regions. When aged regions are +// repeatedly excluded from the collection set, the amount of live memory within the young generation tends to +// accumulate and this has the undesirable side effect of causing young-generation collections to require much more +// CPU and wall-clock time. +// +// A second benefit of treating aged regions differently than other regions during collection set selection is +// that this allows us to more accurately budget memory to hold the results of evacuation. Memory for evacuation +// of aged regions must be reserved in the old generations. Memory for evacuation of all other regions must be +// reserved in the young generation. +// +// A side effect performed by this function is to tally up the number of regions and the number of live bytes +// that we plan to promote-in-place during the current GC cycle. This information, which is stored with +// an invocation of heap->set_promotion_in_place_potential(), feeds into subsequent decisions about when to +// trigger the next GC and may identify special work to be done during this GC cycle if we choose to abbreviate it. +// +// Returns bytes of old-gen memory consumed by selected aged regions +size_t ShenandoahHeuristics::select_aged_regions(size_t old_available, size_t num_regions, + bool candidate_regions_for_promotion_by_copy[]) { ShenandoahHeap* heap = ShenandoahHeap::heap(); assert(heap->mode()->is_generational(), "Only in generational mode"); - + ShenandoahMarkingContext* const ctx = heap->marking_context(); size_t old_consumed = 0; + size_t promo_potential = 0; + size_t anticipated_promote_in_place_live = 0; + + heap->clear_promotion_in_place_potential(); + heap->clear_promotion_potential(); + size_t candidates = 0; + size_t candidates_live = 0; + size_t old_garbage_threshold = (ShenandoahHeapRegion::region_size_bytes() * ShenandoahOldGarbageThreshold) / 100; + size_t promote_in_place_regions = 0; + size_t promote_in_place_live = 0; + size_t promote_in_place_pad = 0; + size_t anticipated_candidates = 0; + size_t anticipated_promote_in_place_regions = 0; + + // Sort the promotion-eligible regions according to live-data-bytes so that we can first reclaim regions that require + // less evacuation effort. This prioritizes garbage first, expanding the allocation pool before we begin the work of + // reclaiming regions that require more effort. + AgedRegionData* sorted_regions = (AgedRegionData*) alloca(num_regions * sizeof(AgedRegionData)); for (size_t i = 0; i < num_regions; i++) { - ShenandoahHeapRegion* region = heap->get_region(i); - if (in_generation(region) && !region->is_empty() && region->is_regular() && (region->age() >= InitialTenuringThreshold)) { - size_t promotion_need = (size_t) (region->get_live_data_bytes() * ShenandoahEvacWaste); - if (old_consumed + promotion_need < old_available) { + ShenandoahHeapRegion* r = heap->get_region(i); + if (r->is_empty() || !r->has_live() || !r->is_young() || !r->is_regular()) { + continue; + } + if (r->age() >= InitialTenuringThreshold) { + r->save_top_before_promote(); + if ((r->garbage() < old_garbage_threshold)) { + HeapWord* tams = ctx->top_at_mark_start(r); + HeapWord* original_top = r->top(); + if (tams == original_top) { + // Fill the remnant memory within this region to assure no allocations prior to promote in place. Otherwise, + // newly allocated objects will not be parseable when promote in place tries to register them. Furthermore, any + // new allocations would not necessarily be eligible for promotion. This addresses both issues. + size_t remnant_size = r->free() / HeapWordSize; + if (remnant_size > ShenandoahHeap::min_fill_size()) { + ShenandoahHeap::fill_with_object(original_top, remnant_size); + r->set_top(r->end()); + promote_in_place_pad += remnant_size * HeapWordSize; + } else { + // Since the remnant is so small that it cannot be filled, we don't have to worry about any accidental + // allocations occuring within this region before the region is promoted in place. + } + promote_in_place_regions++; + promote_in_place_live += r->get_live_data_bytes(); + } + // Else, we do not promote this region (either in place or by copy) because it has received new allocations. + + // During evacuation, we exclude from promotion regions for which age > tenure threshold, garbage < garbage-threshold, + // and get_top_before_promote() != tams + } else { + // After sorting and selecting best candidates below, we may decide to exclude this promotion-eligible region + // from the current collection sets. If this happens, we will consider this region as part of the anticipated + // promotion potential for the next GC pass. + size_t live_data = r->get_live_data_bytes(); + candidates_live += live_data; + sorted_regions[candidates]._region = r; + sorted_regions[candidates++]._live_data = live_data; + } + } else { + // We only anticipate to promote regular regions if garbage() is above threshold. Tenure-aged regions with less + // garbage are promoted in place. These take a different path to old-gen. Note that certain regions that are + // excluded from anticipated promotion because their garbage content is too low (causing us to anticipate that + // the region would be promoted in place) may be eligible for evacuation promotion by the time promotion takes + // place during a subsequent GC pass because more garbage is found within the region between now and then. This + // should not happen if we are properly adapting the tenure age. The theory behind adaptive tenuring threshold + // is to choose the youngest age that demonstrates no "significant" futher loss of population since the previous + // age. If not this, we expect the tenure age to demonstrate linear population decay for at least two population + // samples, whereas we expect to observe exponetial population decay for ages younger than the tenure age. + // + // In the case that certain regions which were anticipated to be promoted in place need to be promoted by + // evacuation, it may be the case that there is not sufficient reserve within old-gen to hold evacuation of + // these regions. The likely outcome is that these regions will not be selected for evacuation or promotion + // in the current cycle and we will anticipate that they will be promoted in the next cycle. This will cause + // us to reserve more old-gen memory so that these objects can be promoted in the subsequent cycle. + // + // TODO: + // If we are auto-tuning the tenure age and regions that were anticipated to be promoted in place end up + // being promoted by evacuation, this event should feed into the tenure-age-selection heuristic so that + // the tenure age can be increased. + if (heap->is_aging_cycle() && (r->age() + 1 == InitialTenuringThreshold)) { + if (r->garbage() >= old_garbage_threshold) { + anticipated_candidates++; + promo_potential += r->get_live_data_bytes(); + } + else { + anticipated_promote_in_place_regions++; + anticipated_promote_in_place_live += r->get_live_data_bytes(); + } + } + } + // Note that we keep going even if one region is excluded from selection. + // Subsequent regions may be selected if they have smaller live data. + } + // Sort in increasing order according to live data bytes. Note that candidates represents the number of regions + // that qualify to be promoted by evacuation. + if (candidates > 0) { + QuickSort::sort(sorted_regions, candidates, compare_by_aged_live, false); + for (size_t i = 0; i < candidates; i++) { + size_t region_live_data = sorted_regions[i]._live_data; + size_t promotion_need = (size_t) (region_live_data * ShenandoahPromoEvacWaste); + if (old_consumed + promotion_need <= old_available) { + ShenandoahHeapRegion* region = sorted_regions[i]._region; old_consumed += promotion_need; - preselected_regions[i] = true; + candidate_regions_for_promotion_by_copy[region->index()] = true; + } else { + // We rejected this promotable region from the collection set because we had no room to hold its copy. + // Add this region to promo potential for next GC. + promo_potential += region_live_data; } - // Note that we keep going even if one region is excluded from selection. - // Subsequent regions may be selected if they have smaller live data. + // We keep going even if one region is excluded from selection because we need to accumulate all eligible + // regions that are not preselected into promo_potential } } + heap->set_pad_for_promote_in_place(promote_in_place_pad); + heap->set_promotion_potential(promo_potential); + heap->set_promotion_in_place_potential(anticipated_promote_in_place_live); return old_consumed; } void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collection_set, ShenandoahOldHeuristics* old_heuristics) { ShenandoahHeap* heap = ShenandoahHeap::heap(); bool is_generational = heap->mode()->is_generational(); + size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); assert(collection_set->count() == 0, "Must be empty"); assert(!is_generational || !_generation->is_old(), "Old GC invokes ShenandoahOldHeuristics::choose_collection_set()"); @@ -113,6 +265,7 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec RegionData* candidates = _region_data; size_t cand_idx = 0; + size_t preselected_candidates = 0; size_t total_garbage = 0; @@ -122,12 +275,21 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec size_t free = 0; size_t free_regions = 0; + size_t old_garbage_threshold = (region_size_bytes * ShenandoahOldGarbageThreshold) / 100; + // This counts number of humongous regions that we intend to promote in this cycle. + size_t humongous_regions_promoted = 0; + // This counts bytes of memory used by hunongous regions to be promoted in place. + size_t humongous_bytes_promoted = 0; + // This counts number of regular regions that will be promoted in place. + size_t regular_regions_promoted_in_place = 0; + // This counts bytes of memory used by regular regions to be promoted in place. + size_t regular_regions_promoted_usage = 0; + for (size_t i = 0; i < num_regions; i++) { ShenandoahHeapRegion* region = heap->get_region(i); if (is_generational && !in_generation(region)) { continue; } - size_t garbage = region->garbage(); total_garbage += garbage; if (region->is_empty()) { @@ -141,17 +303,37 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec region->make_trash_immediate(); } else { assert(!_generation->is_old(), "OLD is handled elsewhere"); + bool is_candidate; // This is our candidate for later consideration. - candidates[cand_idx]._region = region; if (is_generational && collection_set->is_preselected(i)) { - // If region is preselected, we know mode()->is_generational() and region->age() >= InitialTenuringThreshold) - garbage = ShenandoahHeapRegion::region_size_bytes(); + // If !is_generational, we cannot ask if is_preselected. If is_preselected, we know + // region->age() >= InitialTenuringThreshold). + is_candidate = true; + preselected_candidates++; + // Set garbage value to maximum value to force this into the sorted collection set. + garbage = region_size_bytes; + } else if (is_generational && region->is_young() && (region->age() >= InitialTenuringThreshold)) { + // Note that for GLOBAL GC, region may be OLD, and OLD regions do not qualify for pre-selection + + // This region is old enough to be promoted but it was not preselected, either because its garbage is below + // ShenandoahOldGarbageThreshold so it will be promoted in place, or because there is not sufficient room + // in old gen to hold the evacuated copies of this region's live data. In both cases, we choose not to + // place this region into the collection set. + if (region->garbage_before_padded_for_promote() < old_garbage_threshold) { + regular_regions_promoted_in_place++; + regular_regions_promoted_usage += region->used_before_promote(); + } + is_candidate = false; + } else { + is_candidate = true; + } + if (is_candidate) { + candidates[cand_idx]._region = region; + candidates[cand_idx]._u._garbage = garbage; + cand_idx++; } - candidates[cand_idx]._garbage = garbage; - cand_idx++; } } else if (region->is_humongous_start()) { - // Reclaim humongous regions here, and count them as the immediate garbage #ifdef ASSERT bool reg_live = region->has_live(); @@ -166,6 +348,13 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec // Count only the start. Continuations would be counted on "trash" path immediate_regions++; immediate_garbage += garbage; + } else { + if (region->is_young() && region->age() >= InitialTenuringThreshold) { + oop obj = cast_to_oop(region->bottom()); + size_t humongous_regions = ShenandoahHeapRegion::required_regions(obj->size() * HeapWordSize); + humongous_regions_promoted += humongous_regions; + humongous_bytes_promoted += obj->size() * HeapWordSize; + } } } else if (region->is_trash()) { // Count in just trashed collection set, during coalesced CM-with-UR @@ -173,6 +362,14 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec immediate_garbage += garbage; } } + heap->reserve_promotable_humongous_regions(humongous_regions_promoted); + heap->reserve_promotable_humongous_usage(humongous_bytes_promoted); + heap->reserve_promotable_regular_regions(regular_regions_promoted_in_place); + heap->reserve_promotable_regular_usage(regular_regions_promoted_usage); + log_info(gc, ergo)("Planning to promote in place " SIZE_FORMAT " humongous regions and " SIZE_FORMAT + " regular regions, spanning a total of " SIZE_FORMAT " used bytes", + humongous_regions_promoted, regular_regions_promoted_in_place, + humongous_regions_promoted * ShenandoahHeapRegion::region_size_bytes() + regular_regions_promoted_usage); // Step 2. Look back at garbage statistics, and decide if we want to collect anything, // given the amount of immediately reclaimable garbage. If we do, figure out the collection set. @@ -185,7 +382,9 @@ void ShenandoahHeuristics::choose_collection_set(ShenandoahCollectionSet* collec size_t immediate_percent = (total_garbage == 0) ? 0 : (immediate_garbage * 100 / total_garbage); collection_set->set_immediate_trash(immediate_garbage); - if (immediate_percent <= ShenandoahImmediateThreshold) { + ShenandoahGeneration* young_gen = heap->young_generation(); + bool doing_promote_in_place = (humongous_regions_promoted + regular_regions_promoted_in_place > 0); + if (doing_promote_in_place || (preselected_candidates > 0) || (immediate_percent <= ShenandoahImmediateThreshold)) { if (old_heuristics != nullptr) { old_heuristics->prime_collection_set(collection_set); } else { @@ -359,6 +558,12 @@ void ShenandoahHeuristics::initialize() { // Nothing to do by default. } +size_t ShenandoahHeuristics::bytes_of_allocation_runway_before_gc_trigger(size_t young_regions_to_be_recycled) { + assert(false, "Only implemented for young Adaptive Heuristics"); + return 0; +} + + double ShenandoahHeuristics::elapsed_cycle_time() const { return os::elapsedTime() - _cycle_start; } @@ -370,6 +575,10 @@ bool ShenandoahHeuristics::in_generation(ShenandoahHeapRegion* region) { } size_t ShenandoahHeuristics::min_free_threshold() { - size_t min_free_threshold = _generation->is_old() ? ShenandoahOldMinFreeThreshold : ShenandoahMinFreeThreshold; - return _generation->soft_max_capacity() / 100 * min_free_threshold; + assert(!_generation->is_old(), "min_free_threshold is only relevant to young GC"); + size_t min_free_threshold = ShenandoahMinFreeThreshold; + // Note that soft_max_capacity() / 100 * min_free_threshold is smaller than max_capacity() / 100 * min_free_threshold. + // We want to behave conservatively here, so use max_capacity(). By returning a larger value, we cause the GC to + // trigger when the remaining amount of free shrinks below the larger threshold. + return _generation->max_capacity() / 100 * min_free_threshold; } diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp index aba842c24552d..6b33025dfa8e9 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahHeuristics.hpp @@ -70,7 +70,10 @@ class ShenandoahHeuristics : public CHeapObj { typedef struct { ShenandoahHeapRegion* _region; - size_t _garbage; + union { + size_t _garbage; // Not used by old-gen heuristics. + size_t _live_data; // Only used for old-gen heuristics, which prioritizes retention of _live_data over garbage reclaim + } _u; } RegionData; ShenandoahGeneration* _generation; @@ -106,6 +109,14 @@ class ShenandoahHeuristics : public CHeapObj { static int compare_by_garbage(RegionData a, RegionData b); + // Compare by live is used to prioritize compaction of old-gen regions. With old-gen compaction, the goal is + // to tightly pack long-lived objects into available regions. In most cases, there has not been an accumulation + // of garbage within old-gen regions. The more likely opportunity will be to combine multiple sparsely populated + // old-gen regions which may have been promoted in place into a smaller number of densely packed old-gen regions. + // This improves subsequent allocation efficiency and reduces the likelihood of allocation failure (including + // humongous allocation failure) due to fragmentation of the available old-gen allocation pool + static int compare_by_live(RegionData a, RegionData b); + // TODO: We need to enhance this API to give visibility to accompanying old-gen evacuation effort. // In the case that the old-gen evacuation effort is small or zero, the young-gen heuristics // should feel free to dedicate increased efforts to young-gen evacuation. @@ -156,7 +167,7 @@ class ShenandoahHeuristics : public CHeapObj { virtual void reset_gc_learning(); - virtual size_t select_aged_regions(size_t old_available, size_t num_regions, bool* preselected_regions); + virtual size_t select_aged_regions(size_t old_available, size_t num_regions, bool candidate_regions_for_promotion_by_copy[]); virtual void choose_collection_set(ShenandoahCollectionSet* collection_set, ShenandoahOldHeuristics* old_heuristics); @@ -169,6 +180,8 @@ class ShenandoahHeuristics : public CHeapObj { virtual bool is_experimental() = 0; virtual void initialize(); + virtual size_t bytes_of_allocation_runway_before_gc_trigger(size_t region_to_be_recycled); + double elapsed_cycle_time() const; }; diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp index ad8535ebe336d..5722103d73a9f 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp @@ -43,20 +43,23 @@ ShenandoahOldHeuristics::ShenandoahOldHeuristics(ShenandoahOldGeneration* genera _next_old_collection_candidate(0), _last_old_region(0), _trigger_heuristic(trigger_heuristic), + _old_generation(generation), _promotion_failed(false), - _old_generation(generation) + _cannot_expand_trigger(false), + _fragmentation_trigger(false), + _growth_trigger(false) { assert(_generation->is_old(), "This service only available for old-gc heuristics"); } bool ShenandoahOldHeuristics::prime_collection_set(ShenandoahCollectionSet* collection_set) { + ShenandoahHeap* heap = ShenandoahHeap::heap(); if (unprocessed_old_collection_candidates() == 0) { return false; } _first_pinned_candidate = NOT_FOUND; - ShenandoahHeap* heap = ShenandoahHeap::heap(); uint included_old_regions = 0; size_t evacuated_old_bytes = 0; size_t collected_old_bytes = 0; @@ -66,15 +69,36 @@ bool ShenandoahOldHeuristics::prime_collection_set(ShenandoahCollectionSet* coll // of memory that can still be evacuated. We address this by reducing the evacuation budget by the amount // of live memory in that region and by the amount of unallocated memory in that region if the evacuation // budget is constrained by availability of free memory. - size_t old_evacuation_budget = (size_t) ((double) heap->get_old_evac_reserve() / ShenandoahEvacWaste); + size_t old_evacuation_budget = (size_t) ((double) heap->get_old_evac_reserve() / ShenandoahOldEvacWaste); + size_t unfragmented_available = _old_generation->free_unaffiliated_regions() * ShenandoahHeapRegion::region_size_bytes(); + size_t fragmented_available; + size_t excess_fragmented_available; + + if (unfragmented_available > old_evacuation_budget) { + unfragmented_available = old_evacuation_budget; + fragmented_available = 0; + excess_fragmented_available = 0; + } else { + assert(_old_generation->available() > old_evacuation_budget, "Cannot budget more than is available"); + fragmented_available = _old_generation->available() - unfragmented_available; + assert(fragmented_available + unfragmented_available >= old_evacuation_budget, "Budgets do not add up"); + if (fragmented_available + unfragmented_available > old_evacuation_budget) { + excess_fragmented_available = (fragmented_available + unfragmented_available) - old_evacuation_budget; + fragmented_available -= excess_fragmented_available; + } + } + size_t remaining_old_evacuation_budget = old_evacuation_budget; - size_t lost_evacuation_capacity = 0; log_info(gc)("Choose old regions for mixed collection: old evacuation budget: " SIZE_FORMAT "%s, candidates: %u", byte_size_in_proper_unit(old_evacuation_budget), proper_unit_for_byte_size(old_evacuation_budget), unprocessed_old_collection_candidates()); + size_t lost_evacuation_capacity = 0; + // The number of old-gen regions that were selected as candidates for collection at the end of the most recent old-gen - // concurrent marking phase and have not yet been collected is represented by unprocessed_old_collection_candidates() + // concurrent marking phase and have not yet been collected is represented by unprocessed_old_collection_candidates(). + // Candidate regions are ordered according to increasing amount of live data. If there is not sufficient room to + // evacuate region N, then there is no need to even consider evacuating region N+1. while (unprocessed_old_collection_candidates() > 0) { // Old collection candidates are sorted in order of decreasing garbage contained therein. ShenandoahHeapRegion* r = next_old_collection_candidate(); @@ -82,31 +106,74 @@ bool ShenandoahOldHeuristics::prime_collection_set(ShenandoahCollectionSet* coll break; } - // If we choose region r to be collected, then we need to decrease the capacity to hold other evacuations by - // the size of r's free memory. - - // It's probably overkill to compensate with lost_evacuation_capacity. - // But it's the safe thing to do and has minimal impact on content of primed collection set. - size_t live = r->get_live_data_bytes(); - if (live + lost_evacuation_capacity <= remaining_old_evacuation_budget) { - // Decrement remaining evacuation budget by bytes that will be copied. - lost_evacuation_capacity += r->free(); - remaining_old_evacuation_budget -= live; - collection_set->add_region(r); - included_old_regions++; - evacuated_old_bytes += live; - collected_old_bytes += r->garbage(); - consume_old_collection_candidate(); + // If region r is evacuated to fragmented memory (to free memory within a partially used region), then we need + // to decrease the capacity of the fragmented memory by the scaled loss. + + size_t live_data_for_evacuation = r->get_live_data_bytes(); + size_t lost_available = r->free(); + + if ((lost_available > 0) && (excess_fragmented_available > 0)) { + if (lost_available < excess_fragmented_available) { + excess_fragmented_available -= lost_available; + lost_evacuation_capacity -= lost_available; + lost_available = 0; + } else { + lost_available -= excess_fragmented_available; + lost_evacuation_capacity -= excess_fragmented_available; + excess_fragmented_available = 0; + } + } + size_t scaled_loss = (size_t) ((double) lost_available / ShenandoahOldEvacWaste); + if ((lost_available > 0) && (fragmented_available > 0)) { + if (scaled_loss + live_data_for_evacuation < fragmented_available) { + fragmented_available -= scaled_loss; + scaled_loss = 0; + } else { + // We will have to allocate this region's evacuation memory from unfragmented memory, so don't bother + // to decrement scaled_loss + } + } + if (scaled_loss > 0) { + // We were not able to account for the lost free memory within fragmented memory, so we need to take this + // allocation out of unfragmented memory. Unfragmented memory does not need to account for loss of free. + if (live_data_for_evacuation > unfragmented_available) { + // There is not room to evacuate this region or any that come after it in within the candidates array. + break; + } else { + unfragmented_available -= live_data_for_evacuation; + } } else { - break; + // Since scaled_loss == 0, we have accounted for the loss of free memory, so we can allocate from either + // fragmented or unfragmented available memory. Use up the fragmented memory budget first. + size_t evacuation_need = live_data_for_evacuation; + + if (evacuation_need > fragmented_available) { + evacuation_need -= fragmented_available; + fragmented_available = 0; + } else { + fragmented_available -= evacuation_need; + evacuation_need = 0; + } + if (evacuation_need > unfragmented_available) { + // There is not room to evacuate this region or any that come after it in within the candidates array. + break; + } else { + unfragmented_available -= evacuation_need; + // dead code: evacuation_need == 0; + } } + collection_set->add_region(r); + included_old_regions++; + evacuated_old_bytes += live_data_for_evacuation; + collected_old_bytes += r->garbage(); + consume_old_collection_candidate(); } if (_first_pinned_candidate != NOT_FOUND) { // Need to deal with pinned regions slide_pinned_regions_to_front(); } - + decrease_unprocessed_old_collection_candidates_live_memory(evacuated_old_bytes); if (included_old_regions > 0) { log_info(gc)("Old-gen piggyback evac (" UINT32_FORMAT " regions, evacuating " SIZE_FORMAT "%s, reclaiming: " SIZE_FORMAT "%s)", included_old_regions, @@ -116,6 +183,8 @@ bool ShenandoahOldHeuristics::prime_collection_set(ShenandoahCollectionSet* coll if (unprocessed_old_collection_candidates() == 0) { // We have added the last of our collection candidates to a mixed collection. + // Any triggers that occurred during mixed evacuations may no longer be valid. They can retrigger if appropriate. + clear_triggers(); _old_generation->transition_to(ShenandoahOldGeneration::IDLE); } else if (included_old_regions == 0) { // We have candidates, but none were included for evacuation - are they all pinned? @@ -202,7 +271,7 @@ void ShenandoahOldHeuristics::slide_pinned_regions_to_front() { if (skipped._region->is_pinned()) { RegionData& available_slot = _region_data[write_index]; available_slot._region = skipped._region; - available_slot._garbage = skipped._garbage; + available_slot._u._live_data = skipped._u._live_data; --write_index; } } @@ -232,6 +301,7 @@ void ShenandoahOldHeuristics::prepare_for_old_collections() { size_t num_regions = heap->num_regions(); size_t immediate_garbage = 0; size_t immediate_regions = 0; + size_t live_data = 0; RegionData* candidates = _region_data; for (size_t i = 0; i < num_regions; i++) { @@ -241,7 +311,9 @@ void ShenandoahOldHeuristics::prepare_for_old_collections() { } size_t garbage = region->garbage(); + size_t live_bytes = region->get_live_data_bytes(); total_garbage += garbage; + live_data += live_bytes; if (region->is_regular() || region->is_pinned()) { if (!region->has_live()) { @@ -252,7 +324,7 @@ void ShenandoahOldHeuristics::prepare_for_old_collections() { } else { region->begin_preemptible_coalesce_and_fill(); candidates[cand_idx]._region = region; - candidates[cand_idx]._garbage = garbage; + candidates[cand_idx]._u._live_data = live_bytes; cand_idx++; } } else if (region->is_humongous_start()) { @@ -273,42 +345,63 @@ void ShenandoahOldHeuristics::prepare_for_old_collections() { } } + _old_generation->set_live_bytes_after_last_mark(live_data); + // TODO: Consider not running mixed collects if we recovered some threshold percentage of memory from immediate garbage. // This would be similar to young and global collections shortcutting evacuation, though we'd probably want a separate // threshold for the old generation. - // Prioritize regions to select garbage-first regions - QuickSort::sort(candidates, cand_idx, compare_by_garbage, false); + // Unlike young, we are more interested in efficiently packing OLD-gen than in reclaiming garbage first. We sort by live-data. + // Some regular regions may have been promoted in place with no garbage but also with very little live data. When we "compact" + // old-gen, we want to pack these underutilized regions together so we can have more unaffiliated (unfragmented) free regions + // in old-gen. + QuickSort::sort(candidates, cand_idx, compare_by_live, false); - // Any old-gen region that contains (ShenandoahOldGarbageThreshold (default value 25))% garbage or more is to - // be evacuated. + // Any old-gen region that contains (ShenandoahOldGarbageThreshold (default value 25)% garbage or more is to be + // added to the list of candidates for subsequent mixed evacuations. // // TODO: allow ShenandoahOldGarbageThreshold to be determined adaptively, by heuristics. + const size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); + + // The convention is to collect regions that have more than this amount of garbage. + const size_t garbage_threshold = region_size_bytes * ShenandoahOldGarbageThreshold / 100; + + // Englightened interpretation: collect regions that have less than this amount of live. + const size_t live_threshold = region_size_bytes - garbage_threshold; - const size_t garbage_threshold = ShenandoahHeapRegion::region_size_bytes() * ShenandoahOldGarbageThreshold / 100; size_t candidates_garbage = 0; _last_old_region = (uint)cand_idx; _last_old_collection_candidate = (uint)cand_idx; _next_old_collection_candidate = 0; + size_t unfragmented = 0; + for (size_t i = 0; i < cand_idx; i++) { - if (candidates[i]._garbage < garbage_threshold) { - // Candidates are sorted in decreasing order of garbage, so no regions after this will be above the threshold + size_t live = candidates[i]._u._live_data; + if (live > live_threshold) { + // Candidates are sorted in increasing order of live data, so no regions after this will be below the threshold. _last_old_collection_candidate = (uint)i; break; } - candidates_garbage += candidates[i]._garbage; + size_t region_garbage = candidates[i]._region->garbage(); + size_t region_free = candidates[i]._region->free(); + candidates_garbage += region_garbage; + unfragmented += region_free; } // Note that we do not coalesce and fill occupied humongous regions // HR: humongous regions, RR: regular regions, CF: coalesce and fill regions size_t collectable_garbage = immediate_garbage + candidates_garbage; - log_info(gc)("Old-Gen Collectable Garbage: " SIZE_FORMAT "%s over " UINT32_FORMAT " regions, " + size_t old_candidates = _last_old_collection_candidate; + log_info(gc)("Old-Gen Collectable Garbage: " SIZE_FORMAT "%s " + "consolidated with free: " SIZE_FORMAT "%s, over " SIZE_FORMAT " regions, " "Old-Gen Immediate Garbage: " SIZE_FORMAT "%s over " SIZE_FORMAT " regions.", - byte_size_in_proper_unit(collectable_garbage), proper_unit_for_byte_size(collectable_garbage), _last_old_collection_candidate, - byte_size_in_proper_unit(immediate_garbage), proper_unit_for_byte_size(immediate_garbage), immediate_regions); - + byte_size_in_proper_unit(collectable_garbage), proper_unit_for_byte_size(collectable_garbage), + byte_size_in_proper_unit(unfragmented), proper_unit_for_byte_size(unfragmented), old_candidates, + byte_size_in_proper_unit(immediate_garbage), proper_unit_for_byte_size(immediate_garbage), immediate_regions); + size_t mixed_evac_live = old_candidates * region_size_bytes - (candidates_garbage + unfragmented); + set_unprocessed_old_collection_candidates_live_memory(mixed_evac_live); if (unprocessed_old_collection_candidates() == 0) { _old_generation->transition_to(ShenandoahOldGeneration::IDLE); } else { @@ -316,6 +409,20 @@ void ShenandoahOldHeuristics::prepare_for_old_collections() { } } +size_t ShenandoahOldHeuristics::unprocessed_old_collection_candidates_live_memory() const { + return _live_bytes_in_unprocessed_candidates; +} + +void ShenandoahOldHeuristics::set_unprocessed_old_collection_candidates_live_memory(size_t initial_live) { + _live_bytes_in_unprocessed_candidates = initial_live; +} + +void ShenandoahOldHeuristics::decrease_unprocessed_old_collection_candidates_live_memory(size_t evacuated_live) { + assert(evacuated_live <= _live_bytes_in_unprocessed_candidates, "Cannot evacuate more than was present"); + _live_bytes_in_unprocessed_candidates -= evacuated_live; +} + + // TODO: Unused? uint ShenandoahOldHeuristics::last_old_collection_candidate_index() { return _last_old_collection_candidate; @@ -356,7 +463,7 @@ unsigned int ShenandoahOldHeuristics::get_coalesce_and_fill_candidates(Shenandoa while (index < end) { *buffer++ = _region_data[index++]._region; } - return _last_old_region - _next_old_collection_candidate; + return (_last_old_region - _next_old_collection_candidate); } void ShenandoahOldHeuristics::abandon_collection_candidates() { @@ -366,24 +473,31 @@ void ShenandoahOldHeuristics::abandon_collection_candidates() { } void ShenandoahOldHeuristics::handle_promotion_failure() { - if (!_promotion_failed) { - if (ShenandoahHeap::heap()->generation_sizer()->transfer_capacity(_old_generation)) { - log_info(gc)("Increased size of old generation due to promotion failure."); - } - // TODO: Increase tenuring threshold to push back on promotions. - } _promotion_failed = true; } void ShenandoahOldHeuristics::record_cycle_start() { - _promotion_failed = false; _trigger_heuristic->record_cycle_start(); } void ShenandoahOldHeuristics::record_cycle_end() { _trigger_heuristic->record_cycle_end(); + clear_triggers(); +} + +void ShenandoahOldHeuristics::trigger_old_has_grown() { + _growth_trigger = true; } + +void ShenandoahOldHeuristics::clear_triggers() { + // Clear any triggers that were set during mixed evacuations. Conditions may be different now that this phase has finished. + _promotion_failed = false; + _cannot_expand_trigger = false; + _fragmentation_trigger = false; + _growth_trigger = false; + } + bool ShenandoahOldHeuristics::should_start_gc() { // Cannot start a new old-gen GC until previous one has finished. // @@ -393,13 +507,49 @@ bool ShenandoahOldHeuristics::should_start_gc() { return false; } - // If there's been a promotion failure (and we don't have regions already scheduled for evacuation), - // start a new old generation collection. - if (_promotion_failed) { - log_info(gc)("Trigger: Promotion Failure"); + if (_cannot_expand_trigger) { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + size_t old_gen_capacity = _old_generation->max_capacity(); + size_t heap_capacity = heap->capacity(); + double percent = 100.0 * ((double) old_gen_capacity) / heap_capacity; + log_info(gc)("Trigger (OLD): Expansion failure, current size: " SIZE_FORMAT "%s which is %.1f%% of total heap size", + byte_size_in_proper_unit(old_gen_capacity), proper_unit_for_byte_size(old_gen_capacity), percent); return true; } + if (_fragmentation_trigger) { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + size_t used = _old_generation->used(); + size_t used_regions_size = _old_generation->used_regions_size(); + size_t used_regions = _old_generation->used_regions(); + assert(used_regions_size > used_regions, "Cannot have more used than used regions"); + size_t fragmented_free = used_regions_size - used; + double percent = 100.0 * ((double) fragmented_free) / used_regions_size; + log_info(gc)("Trigger (OLD): Old has become fragmented: " + SIZE_FORMAT "%s available bytes spread between " SIZE_FORMAT " regions (%.1f%% free)", + byte_size_in_proper_unit(fragmented_free), proper_unit_for_byte_size(fragmented_free), used_regions, percent); + return true; + } + + if (_growth_trigger) { + // Growth may be falsely triggered during mixed evacuations, before the mixed-evacuation candidates have been + // evacuated. Before acting on a false trigger, we check to confirm the trigger condition is still satisfied. + ShenandoahHeap* heap = ShenandoahHeap::heap(); + size_t current_usage = _old_generation->used(); + size_t trigger_threshold = _old_generation->usage_trigger_threshold(); + if (current_usage > trigger_threshold) { + size_t live_at_previous_old = _old_generation->get_live_bytes_after_last_mark(); + double percent_growth = 100.0 * ((double) current_usage - live_at_previous_old) / live_at_previous_old; + log_info(gc)("Trigger (OLD): Old has overgrown, live at end of previous OLD marking: " + SIZE_FORMAT "%s, current usage: " SIZE_FORMAT "%s, percent growth: %.1f%%", + byte_size_in_proper_unit(live_at_previous_old), proper_unit_for_byte_size(live_at_previous_old), + byte_size_in_proper_unit(current_usage), proper_unit_for_byte_size(current_usage), percent_growth); + return true; + } else { + _growth_trigger = false; + } + } + // Otherwise, defer to configured heuristic for gc trigger. return _trigger_heuristic->should_start_gc(); } @@ -409,14 +559,20 @@ bool ShenandoahOldHeuristics::should_degenerate_cycle() { } void ShenandoahOldHeuristics::record_success_concurrent(bool abbreviated) { + // Forget any triggers that occured while OLD GC was ongoing. If we really need to start another, it will retrigger. + clear_triggers(); _trigger_heuristic->record_success_concurrent(abbreviated); } void ShenandoahOldHeuristics::record_success_degenerated() { + // Forget any triggers that occured while OLD GC was ongoing. If we really need to start another, it will retrigger. + clear_triggers(); _trigger_heuristic->record_success_degenerated(); } void ShenandoahOldHeuristics::record_success_full() { + // Forget any triggers that occured while OLD GC was ongoing. If we really need to start another, it will retrigger. + clear_triggers(); _trigger_heuristic->record_success_full(); } diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp index f8581c2cd01a5..d6789cb87ecb2 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.hpp @@ -69,15 +69,24 @@ class ShenandoahOldHeuristics : public ShenandoahHeuristics { // the end of old final mark. uint _last_old_region; + // How much live data must be evacuated from within the unprocessed mixed evacuation candidates? + size_t _live_bytes_in_unprocessed_candidates; + // This can be the 'static' or 'adaptive' heuristic. ShenandoahHeuristics* _trigger_heuristic; + // Keep a pointer to our generation that we can use without down casting a protected member from the base class. + ShenandoahOldGeneration* _old_generation; + // Flag is set when promotion failure is detected (by gc thread), and cleared when // old generation collection begins (by control thread). volatile bool _promotion_failed; - // Keep a pointer to our generation that we can use without down casting a protected member from the base class. - ShenandoahOldGeneration* _old_generation; + // Flags are set when promotion failure is detected (by gc thread), and cleared when + // old generation collection begins (by control thread). Flags are set and cleared at safepoints. + bool _cannot_expand_trigger; + bool _fragmentation_trigger; + bool _growth_trigger; protected: virtual void choose_collection_set_from_regiondata(ShenandoahCollectionSet* set, RegionData* data, size_t data_size, @@ -97,6 +106,13 @@ class ShenandoahOldHeuristics : public ShenandoahHeuristics { // How many old-collection candidates have not yet been processed? uint unprocessed_old_collection_candidates(); + // How much live memory must be evacuated from within old-collection candidates that have not yet been processed? + size_t unprocessed_old_collection_candidates_live_memory() const; + + void set_unprocessed_old_collection_candidates_live_memory(size_t initial_live); + + void decrease_unprocessed_old_collection_candidates_live_memory(size_t evacuated_live); + // How many old or hidden collection candidates have not yet been processed? uint last_old_collection_candidate_index(); @@ -122,11 +138,17 @@ class ShenandoahOldHeuristics : public ShenandoahHeuristics { // held by this heuristic for supplying mixed collections. void abandon_collection_candidates(); - // Notify the heuristic of promotion failures. The promotion attempt will be skipped and the object will - // be evacuated into the young generation. The collection should complete normally, but we want to schedule - // an old collection as soon as possible. + // Promotion failure does not currently trigger old-gen collections. Often, promotion failures occur because + // old-gen is sized too small rather than because it is necessary to collect old gen. We keep the method + // here in case we decide to feed this signal to sizing or triggering heuristics in the future. void handle_promotion_failure(); + void trigger_cannot_expand() { _cannot_expand_trigger = true; }; + void trigger_old_is_fragmented() { _fragmentation_trigger = true; } + void trigger_old_has_grown(); + + void clear_triggers(); + virtual void record_cycle_start() override; virtual void record_cycle_end() override; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp index c797ad21a7dd3..736354e928219 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.cpp @@ -90,13 +90,13 @@ void ShenandoahCollectionSet::add_region(ShenandoahHeapRegion* r) { assert(!r->is_humongous(), "Only add regular regions to the collection set"); _cset_map[r->index()] = 1; - - size_t live = r->get_live_data_bytes(); + size_t live = r->get_live_data_bytes(); size_t garbage = r->garbage(); - + size_t free = r->free(); if (r->is_young()) { _young_region_count++; _young_bytes_to_evacuate += live; + _young_available_bytes_collected += free; if (r->age() >= InitialTenuringThreshold) { _young_bytes_to_promote += live; } @@ -104,6 +104,7 @@ void ShenandoahCollectionSet::add_region(ShenandoahHeapRegion* r) { _old_region_count++; _old_bytes_to_evacuate += live; _old_garbage += garbage; + _old_available_bytes_collected += free; } _region_count++; @@ -117,6 +118,7 @@ void ShenandoahCollectionSet::add_region(ShenandoahHeapRegion* r) { void ShenandoahCollectionSet::clear() { assert(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "Must be at a safepoint"); + Copy::zero_to_bytes(_cset_map, _map_size); #ifdef ASSERT @@ -140,6 +142,9 @@ void ShenandoahCollectionSet::clear() { _old_region_count = 0; _old_bytes_to_evacuate = 0; + _young_available_bytes_collected = 0; + _old_available_bytes_collected = 0; + _has_old_regions = false; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp index 112fff383ba25..80a5e9dc02827 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectionSet.hpp @@ -66,6 +66,11 @@ class ShenandoahCollectionSet : public CHeapObj { // spans of time while collection set is being constructed. bool* _preselected_regions; + // When a region having memory available to be allocated is added to the collection set, the region's available memory + // should be subtracted from what's available. + size_t _young_available_bytes_collected; + size_t _old_available_bytes_collected; + shenandoah_padding(0); volatile size_t _current_index; shenandoah_padding(1); @@ -111,6 +116,10 @@ class ShenandoahCollectionSet : public CHeapObj { inline size_t get_young_bytes_to_be_promoted(); + size_t get_young_available_bytes_collected() { return _young_available_bytes_collected; } + + size_t get_old_available_bytes_collected() { return _old_available_bytes_collected; } + inline size_t get_old_region_count(); inline size_t get_young_region_count(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.cpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.cpp index 73e1d5dbf2cf8..918aff26add25 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.cpp @@ -128,6 +128,12 @@ bool ShenandoahCollectorPolicy::is_at_shutdown() { return _in_shutdown.is_set(); } +// This may be called by mutator threads. We declare _success_full_gcs volatile to force the value not to be cached +// in a local register or variable by a mutator thread that is checking this value in a loop. +size_t ShenandoahCollectorPolicy::get_fullgc_count() { + return _success_full_gcs + _alloc_failure_degenerated_upgrade_to_full; +} + void ShenandoahCollectorPolicy::print_gc_stats(outputStream* out) const { out->print_cr("Under allocation pressure, concurrent cycles may cancel, and either continue cycle"); out->print_cr("under stop-the-world pause or result in stop-the-world Full GC. Increase heap size,"); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp b/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp index 3ca9965dbda89..244663e6ae31c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahCollectorPolicy.hpp @@ -45,9 +45,9 @@ class ShenandoahCollectorPolicy : public CHeapObj { size_t _success_old_gcs; size_t _interrupted_old_gcs; size_t _success_degenerated_gcs; - size_t _success_full_gcs; + volatile size_t _success_full_gcs; size_t _alloc_failure_degenerated; - size_t _alloc_failure_degenerated_upgrade_to_full; + volatile size_t _alloc_failure_degenerated_upgrade_to_full; size_t _alloc_failure_full; size_t _explicit_concurrent; size_t _explicit_full; @@ -88,6 +88,8 @@ class ShenandoahCollectorPolicy : public CHeapObj { size_t cycle_counter() const; + size_t get_fullgc_count(); + void print_gc_stats(outputStream* out) const; }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp index 6c165ade4e3cc..b09b736ce6e27 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentGC.cpp @@ -217,28 +217,56 @@ bool ShenandoahConcurrentGC::collect(GCCause::Cause cause) { _abbreviated = true; } + // We defer generation resizing actions until after cset regions have been recycled. We do this even following an + // abbreviated cycle. if (heap->mode()->is_generational()) { + bool success; + size_t region_xfer; + const char* region_destination; + ShenandoahYoungGeneration* young_gen = heap->young_generation(); + ShenandoahGeneration* old_gen = heap->old_generation(); { - ShenandoahYoungGeneration* young_gen = heap->young_generation(); - ShenandoahGeneration* old_gen = heap->old_generation(); ShenandoahHeapLocker locker(heap->lock()); + size_t old_region_surplus = heap->get_old_region_surplus(); + size_t old_region_deficit = heap->get_old_region_deficit(); + if (old_region_surplus) { + success = heap->generation_sizer()->transfer_to_young(old_region_surplus); + region_destination = "young"; + region_xfer = old_region_surplus; + } else if (old_region_deficit) { + success = heap->generation_sizer()->transfer_to_old(old_region_deficit); + region_destination = "old"; + region_xfer = old_region_deficit; + if (!success) { + ((ShenandoahOldHeuristics *) old_gen->heuristics())->trigger_cannot_expand(); + } + } else { + region_destination = "none"; + region_xfer = 0; + success = true; + } + heap->set_old_region_surplus(0); + heap->set_old_region_deficit(0); + size_t old_usage_before_evac = heap->capture_old_usage(0); size_t old_usage_now = old_gen->used(); size_t promoted_bytes = old_usage_now - old_usage_before_evac; heap->set_previous_promotion(promoted_bytes); - - young_gen->unadjust_available(); - old_gen->unadjust_available(); - // No need to old_gen->increase_used(). - // That was done when plabs were allocated, accounting for both old evacs and promotions. - - heap->set_alloc_supplement_reserve(0); heap->set_young_evac_reserve(0); heap->set_old_evac_reserve(0); heap->reset_old_evac_expended(); heap->set_promoted_reserve(0); } + + // Report outside the heap lock + size_t young_available = young_gen->available(); + size_t old_available = old_gen->available(); + log_info(gc, ergo)("After cleanup, %s " SIZE_FORMAT " regions to %s to prepare for next gc, old available: " + SIZE_FORMAT "%s, young_available: " SIZE_FORMAT "%s", + success? "successfully transferred": "failed to transfer", region_xfer, region_destination, + byte_size_in_proper_unit(old_available), proper_unit_for_byte_size(old_available), + byte_size_in_proper_unit(young_available), proper_unit_for_byte_size(young_available)); } return true; } @@ -733,64 +761,95 @@ void ShenandoahConcurrentGC::op_final_mark() { // been set aside to hold objects evacuated from the young-gen collection set. Conservatively, this value // equals the entire amount of live young-gen memory within the collection set, even though some of this memory // will likely be promoted. - // - // heap->get_alloc_supplement_reserve() represents the amount of old-gen memory that can be allocated during evacuation - // and update-refs phases of gc. The young evacuation reserve has already been removed from this quantity. // Has to be done after cset selection heap->prepare_concurrent_roots(); - if (!heap->collection_set()->is_empty()) { - LogTarget(Debug, gc, cset) lt; - if (lt.is_enabled()) { - ResourceMark rm; - LogStream ls(lt); - heap->collection_set()->print_on(&ls); - } + if (heap->mode()->is_generational()) { + ShenandoahGeneration* young_gen = heap->young_generation(); + size_t humongous_regions_promoted = heap->get_promotable_humongous_regions(); + size_t regular_regions_promoted_in_place = heap->get_regular_regions_promoted_in_place(); + if (!heap->collection_set()->is_empty() || (humongous_regions_promoted + regular_regions_promoted_in_place > 0)) { + // Even if the collection set is empty, we need to do evacuation if there are regions to be promoted in place. + // Concurrent evacuation takes responsibility for registering objects and setting the remembered set cards to dirty. + + LogTarget(Debug, gc, cset) lt; + if (lt.is_enabled()) { + ResourceMark rm; + LogStream ls(lt); + heap->collection_set()->print_on(&ls); + } - if (ShenandoahVerify) { - heap->verifier()->verify_before_evacuation(); - } + if (ShenandoahVerify) { + heap->verifier()->verify_before_evacuation(); + } + // TODO: we do not need to run update-references following evacuation if collection_set->is_empty(). - heap->set_evacuation_in_progress(true); - // From here on, we need to update references. - heap->set_has_forwarded_objects(true); + heap->set_evacuation_in_progress(true); + // From here on, we need to update references. + heap->set_has_forwarded_objects(true); - // Verify before arming for concurrent processing. - // Otherwise, verification can trigger stack processing. - if (ShenandoahVerify) { - heap->verifier()->verify_during_evacuation(); - } + // Verify before arming for concurrent processing. + // Otherwise, verification can trigger stack processing. + if (ShenandoahVerify) { + heap->verifier()->verify_during_evacuation(); + } - // Arm nmethods/stack for concurrent processing - ShenandoahCodeRoots::arm_nmethods(); - ShenandoahStackWatermark::change_epoch_id(); - - if (heap->mode()->is_generational()) { - // Calculate the temporary evacuation allowance supplement to young-gen memory capacity (for allocations - // and young-gen evacuations). - intptr_t adjustment = heap->get_alloc_supplement_reserve(); - size_t young_available = heap->young_generation()->adjust_available(adjustment); - // old_available is memory that can hold promotions and evacuations. Subtract out the memory that is being - // loaned for young-gen allocations or evacuations. - size_t old_available = heap->old_generation()->adjust_available(-adjustment); - - log_info(gc, ergo)("After generational memory budget adjustments, old available: " SIZE_FORMAT - "%s, young_available: " SIZE_FORMAT "%s", - byte_size_in_proper_unit(old_available), proper_unit_for_byte_size(old_available), - byte_size_in_proper_unit(young_available), proper_unit_for_byte_size(young_available)); - } + // Arm nmethods/stack for concurrent processing + ShenandoahCodeRoots::arm_nmethods(); + ShenandoahStackWatermark::change_epoch_id(); - if (ShenandoahPacing) { - heap->pacer()->setup_for_evac(); + if (ShenandoahPacing) { + heap->pacer()->setup_for_evac(); + } + } else { + if (ShenandoahVerify) { + heap->verifier()->verify_after_concmark(); + } + + if (VerifyAfterGC) { + Universe::verify(); + } } } else { - if (ShenandoahVerify) { - heap->verifier()->verify_after_concmark(); - } + // Not is_generational() + if (!heap->collection_set()->is_empty()) { + LogTarget(Info, gc, ergo) lt; + if (lt.is_enabled()) { + ResourceMark rm; + LogStream ls(lt); + heap->collection_set()->print_on(&ls); + } - if (VerifyAfterGC) { - Universe::verify(); + if (ShenandoahVerify) { + heap->verifier()->verify_before_evacuation(); + } + + heap->set_evacuation_in_progress(true); + // From here on, we need to update references. + heap->set_has_forwarded_objects(true); + + // Verify before arming for concurrent processing. + // Otherwise, verification can trigger stack processing. + if (ShenandoahVerify) { + heap->verifier()->verify_during_evacuation(); + } + + // Arm nmethods/stack for concurrent processing + ShenandoahCodeRoots::arm_nmethods(); + ShenandoahStackWatermark::change_epoch_id(); + + if (ShenandoahPacing) { + heap->pacer()->setup_for_evac(); + } + } else { + if (ShenandoahVerify) { + heap->verifier()->verify_after_concmark(); + } + + if (VerifyAfterGC) { + Universe::verify(); + } } } } @@ -812,6 +871,7 @@ ShenandoahConcurrentEvacThreadClosure::ShenandoahConcurrentEvacThreadClosure(Oop void ShenandoahConcurrentEvacThreadClosure::do_thread(Thread* thread) { JavaThread* const jt = JavaThread::cast(thread); StackWatermarkSet::finish_processing(jt, _oops, StackWatermarkKind::gc); + ShenandoahThreadLocalData::enable_plab_promotions(thread); } class ShenandoahConcurrentEvacUpdateThreadTask : public WorkerTask { @@ -825,6 +885,9 @@ class ShenandoahConcurrentEvacUpdateThreadTask : public WorkerTask { } void work(uint worker_id) { + Thread* worker_thread = Thread::current(); + ShenandoahThreadLocalData::enable_plab_promotions(worker_thread); + // ShenandoahEvacOOMScope has to be setup by ShenandoahContextEvacuateUpdateRootsClosure. // Otherwise, may deadlock with watermark lock ShenandoahContextEvacuateUpdateRootsClosure oops_cl; @@ -1200,7 +1263,6 @@ void ShenandoahConcurrentGC::op_final_updaterefs() { Universe::verify(); } - heap->adjust_generation_sizes(); heap->rebuild_free_set(true /*concurrent*/); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp index 144d9095b0b3c..09e15a0956858 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.cpp @@ -113,6 +113,7 @@ void ShenandoahControlThread::run_service() { // degenerated cycle should be 'promoted' to a full cycle. The decision to // trigger a cycle or not is evaluated on the regulator thread. ShenandoahHeuristics* global_heuristics = heap->global_generation()->heuristics(); + bool old_bootstrap_requested = false; while (!in_graceful_shutdown() && !should_terminate()) { // Figure out if we have pending requests. bool alloc_failure_pending = _alloc_failure_gc.is_set(); @@ -207,10 +208,19 @@ void ShenandoahControlThread::run_service() { // the heuristic to run a young collection so that we can evacuate some old regions. assert(!heap->is_concurrent_old_mark_in_progress(), "Should not be running mixed collections and concurrent marking"); generation = YOUNG; + } else if (_requested_generation == OLD && !old_bootstrap_requested) { + // Arrange to perform a young GC immediately followed by a bootstrap OLD GC. OLD GC typically requires more + // than twice the time required for YOUNG GC, so we run a YOUNG GC to replenish the YOUNG allocation pool before + // we start the longer OLD GC effort. + old_bootstrap_requested = true; + generation = YOUNG; } else { + // if (old_bootstrap_requested && (_requested_generation == OLD)), this starts the bootstrap GC that + // immediately follows the preparatory young GC. + // But we will abandon the planned bootstrap GC if a GLOBAL GC has been now been requested. generation = _requested_generation; + old_bootstrap_requested = false; } - // preemption was requested or this is a regular cycle cause = GCCause::_shenandoah_concurrent_gc; set_gc_mode(default_mode); @@ -391,10 +401,18 @@ void ShenandoahControlThread::run_service() { // Don't wait around if there was an allocation failure - start the next cycle immediately. if (!is_alloc_failure_gc()) { - // The timed wait is necessary because this thread has a responsibility to send - // 'alloc_words' to the pacer when it does not perform a GC. - MonitorLocker lock(&_control_lock, Mutex::_no_safepoint_check_flag); - lock.wait(ShenandoahControlIntervalMax); + if (old_bootstrap_requested) { + _requested_generation = OLD; + _requested_gc_cause = GCCause::_shenandoah_concurrent_gc; + } else { + // The timed wait is necessary because this thread has a responsibility to send + // 'alloc_words' to the pacer when it does not perform a GC. + MonitorLocker lock(&_control_lock, Mutex::_no_safepoint_check_flag); + lock.wait(ShenandoahControlIntervalMax); + } + } else { + // in case of alloc_failure, abandon any plans to do immediate OLD Bootstrap + old_bootstrap_requested = false; } } @@ -457,10 +475,11 @@ void ShenandoahControlThread::process_phase_timings(const ShenandoahHeap* heap) // | v v | // +---> Global Degen +--------------------> Full <----+ // -void ShenandoahControlThread::service_concurrent_normal_cycle(const ShenandoahHeap* heap, +void ShenandoahControlThread::service_concurrent_normal_cycle(ShenandoahHeap* heap, const ShenandoahGenerationType generation, GCCause::Cause cause) { GCIdMark gc_id_mark; + ShenandoahGeneration* the_generation = nullptr; switch (generation) { case YOUNG: { // Run a young cycle. This might or might not, have interrupted an ongoing @@ -469,44 +488,34 @@ void ShenandoahControlThread::service_concurrent_normal_cycle(const ShenandoahHe // they end up in, but we have to be sure we don't promote into any regions // that are in the cset. log_info(gc, ergo)("Start GC cycle (YOUNG)"); - service_concurrent_cycle(heap->young_generation(), cause, false); + the_generation = heap->young_generation(); + service_concurrent_cycle(the_generation, cause, false); break; } case OLD: { log_info(gc, ergo)("Start GC cycle (OLD)"); + the_generation = heap->old_generation(); service_concurrent_old_cycle(heap, cause); break; } case GLOBAL_GEN: { log_info(gc, ergo)("Start GC cycle (GLOBAL)"); - service_concurrent_cycle(heap->global_generation(), cause, false); + the_generation = heap->global_generation(); + service_concurrent_cycle(the_generation, cause, false); break; } case GLOBAL_NON_GEN: { log_info(gc, ergo)("Start GC cycle"); - service_concurrent_cycle(heap->global_generation(), cause, false); + the_generation = heap->global_generation(); + service_concurrent_cycle(the_generation, cause, false); break; } default: ShouldNotReachHere(); } - const char* msg; - if (heap->mode()->is_generational()) { - if (heap->cancelled_gc()) { - msg = (generation == YOUNG) ? "At end of Interrupted Concurrent Young GC" : - "At end of Interrupted Concurrent Bootstrap GC"; - } else { - msg = (generation == YOUNG) ? "At end of Concurrent Young GC" : - "At end of Concurrent Bootstrap GC"; - } - } else { - msg = heap->cancelled_gc() ? "At end of cancelled GC" : - "At end of GC"; - } - heap->log_heap_status(msg); } -void ShenandoahControlThread::service_concurrent_old_cycle(const ShenandoahHeap* heap, GCCause::Cause &cause) { +void ShenandoahControlThread::service_concurrent_old_cycle(ShenandoahHeap* heap, GCCause::Cause &cause) { ShenandoahOldGeneration* old_generation = heap->old_generation(); ShenandoahYoungGeneration* young_generation = heap->young_generation(); ShenandoahOldGeneration::State original_state = old_generation->state(); @@ -562,7 +571,7 @@ void ShenandoahControlThread::service_concurrent_old_cycle(const ShenandoahHeap* set_gc_mode(bootstrapping_old); young_generation->set_old_gen_task_queues(old_generation->task_queues()); ShenandoahGCSession session(cause, young_generation); - service_concurrent_cycle(heap,young_generation, cause, true); + service_concurrent_cycle(heap, young_generation, cause, true); process_phase_timings(heap); if (heap->cancelled_gc()) { // Young generation bootstrap cycle has failed. Concurrent mark for old generation @@ -588,9 +597,13 @@ void ShenandoahControlThread::service_concurrent_old_cycle(const ShenandoahHeap* if (marking_complete) { assert(old_generation->state() != ShenandoahOldGeneration::MARKING, "Should not still be marking"); if (original_state == ShenandoahOldGeneration::MARKING) { + heap->mmu_tracker()->record_old_marking_increment(old_generation, GCId::current(), true, + heap->collection_set()->has_old_regions()); heap->log_heap_status("At end of Concurrent Old Marking finishing increment"); } } else if (original_state == ShenandoahOldGeneration::MARKING) { + heap->mmu_tracker()->record_old_marking_increment(old_generation, GCId::current(), false, + heap->collection_set()->has_old_regions()); heap->log_heap_status("At end of Concurrent Old Marking increment"); } break; @@ -697,7 +710,7 @@ void ShenandoahControlThread::service_concurrent_cycle(ShenandoahGeneration* gen service_concurrent_cycle(heap, generation, cause, do_old_gc_bootstrap); } -void ShenandoahControlThread::service_concurrent_cycle(const ShenandoahHeap* heap, +void ShenandoahControlThread::service_concurrent_cycle(ShenandoahHeap* heap, ShenandoahGeneration* generation, GCCause::Cause& cause, bool do_old_gc_bootstrap) { @@ -713,6 +726,32 @@ void ShenandoahControlThread::service_concurrent_cycle(const ShenandoahHeap* hea // collection. Same for global collections. _degen_generation = generation; } + const char* msg; + if (heap->mode()->is_generational()) { + if (heap->cancelled_gc()) { + msg = (generation->is_young()) ? "At end of Interrupted Concurrent Young GC" : + "At end of Interrupted Concurrent Bootstrap GC"; + } else { + msg = (generation->is_young()) ? "At end of Concurrent Young GC" : + "At end of Concurrent Bootstrap GC"; + // We only record GC results if GC was successful + ShenandoahMmuTracker* mmu_tracker = heap->mmu_tracker(); + if (generation->is_young()) { + if (heap->collection_set()->has_old_regions()) { + bool mixed_is_done = (heap->old_heuristics()->unprocessed_old_collection_candidates() == 0); + mmu_tracker->record_mixed(generation, get_gc_id(), mixed_is_done); + } else { + mmu_tracker->record_young(generation, get_gc_id()); + } + } else { + mmu_tracker->record_bootstrap(generation, get_gc_id(), heap->collection_set()->has_old_regions()); + } + } + } else { + msg = heap->cancelled_gc() ? "At end of cancelled GC" : + "At end of GC"; + } + heap->log_heap_status(msg); } bool ShenandoahControlThread::check_cancellation_or_degen(ShenandoahGC::ShenandoahDegenPoint point) { @@ -928,7 +967,6 @@ void ShenandoahControlThread::handle_alloc_failure(ShenandoahAllocRequest& req) log_info(gc)("Failed to allocate %s, " SIZE_FORMAT "%s", req.type_string(), byte_size_in_proper_unit(req.size() * HeapWordSize), proper_unit_for_byte_size(req.size() * HeapWordSize)); - // Now that alloc failure GC is scheduled, we can abort everything else heap->cancel_gc(GCCause::_allocation_failure); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp index f9596feeeb7bd..339cfa7d5de31 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahControlThread.hpp @@ -88,6 +88,7 @@ class ShenandoahControlThread: public ConcurrentGCThread { ShenandoahSharedFlag _preemption_requested; ShenandoahSharedFlag _gc_requested; ShenandoahSharedFlag _alloc_failure_gc; + ShenandoahSharedFlag _humongous_alloc_failure_gc; ShenandoahSharedFlag _graceful_shutdown; ShenandoahSharedFlag _do_counters_update; ShenandoahSharedFlag _force_counters_update; @@ -173,11 +174,11 @@ class ShenandoahControlThread: public ConcurrentGCThread { void prepare_for_graceful_shutdown(); bool in_graceful_shutdown(); - void service_concurrent_normal_cycle(const ShenandoahHeap* heap, + void service_concurrent_normal_cycle(ShenandoahHeap* heap, const ShenandoahGenerationType generation, GCCause::Cause cause); - void service_concurrent_old_cycle(const ShenandoahHeap* heap, + void service_concurrent_old_cycle(ShenandoahHeap* heap, GCCause::Cause &cause); void set_gc_mode(GCMode new_mode); @@ -191,7 +192,7 @@ class ShenandoahControlThread: public ConcurrentGCThread { static const char* gc_mode_name(GCMode mode); void notify_control_thread(); - void service_concurrent_cycle(const ShenandoahHeap* heap, + void service_concurrent_cycle(ShenandoahHeap* heap, ShenandoahGeneration* generation, GCCause::Cause &cause, bool do_old_gc_bootstrap); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp index fa62f7a9fe8e9..0a7354b17bef2 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.cpp @@ -57,7 +57,11 @@ bool ShenandoahDegenGC::collect(GCCause::Cause cause) { vmop_degenerated(); ShenandoahHeap* heap = ShenandoahHeap::heap(); if (heap->mode()->is_generational()) { - heap->log_heap_status("At end of Degenerated GC"); + bool is_bootstrap_gc = heap->is_concurrent_old_mark_in_progress() && _generation->is_young(); + heap->mmu_tracker()->record_degenerated(_generation, GCId::current(), is_bootstrap_gc, + !heap->collection_set()->has_old_regions()); + const char* msg = is_bootstrap_gc? "At end of Degenerated Boostrap Old GC": "At end of Degenerated GC"; + heap->log_heap_status(msg); } return true; } @@ -272,27 +276,53 @@ void ShenandoahDegenGC::op_degenerated() { } op_cleanup_complete(); + // We defer generation resizing actions until after cset regions have been recycled. + if (heap->mode()->is_generational()) { + size_t old_region_surplus = heap->get_old_region_surplus(); + size_t old_region_deficit = heap->get_old_region_deficit(); + bool success; + size_t region_xfer; + const char* region_destination; + if (old_region_surplus) { + region_xfer = old_region_surplus; + region_destination = "young"; + success = heap->generation_sizer()->transfer_to_young(old_region_surplus); + } else if (old_region_deficit) { + region_xfer = old_region_surplus; + region_destination = "old"; + success = heap->generation_sizer()->transfer_to_old(old_region_deficit); + if (!success) { + ((ShenandoahOldHeuristics *) heap->old_generation()->heuristics())->trigger_cannot_expand(); + } + } else { + region_destination = "none"; + region_xfer = 0; + success = true; + } + + size_t young_available = heap->young_generation()->available(); + size_t old_available = heap->old_generation()->available(); + log_info(gc, ergo)("After cleanup, %s " SIZE_FORMAT " regions to %s to prepare for next gc, old available: " + SIZE_FORMAT "%s, young_available: " SIZE_FORMAT "%s", + success? "successfully transferred": "failed to transfer", region_xfer, region_destination, + byte_size_in_proper_unit(old_available), proper_unit_for_byte_size(old_available), + byte_size_in_proper_unit(young_available), proper_unit_for_byte_size(young_available)); + + heap->set_old_region_surplus(0); + heap->set_old_region_deficit(0); + } break; default: ShouldNotReachHere(); } if (heap->mode()->is_generational()) { - // In case degeneration interrupted concurrent evacuation or update references, - // we need to clean up transient state. Otherwise, these actions have no effect. - - heap->young_generation()->unadjust_available(); - heap->old_generation()->unadjust_available(); - // No need to old_gen->increase_used(). That was done when plabs were allocated, - // accounting for both old evacs and promotions. - - heap->set_alloc_supplement_reserve(0); + // In case degeneration interrupted concurrent evacuation or update references, we need to clean up transient state. + // Otherwise, these actions have no effect. heap->set_young_evac_reserve(0); heap->set_old_evac_reserve(0); heap->reset_old_evac_expended(); heap->set_promoted_reserve(0); - - heap->adjust_generation_sizes(); } if (ShenandoahVerify) { @@ -354,7 +384,16 @@ void ShenandoahDegenGC::op_prepare_evacuation() { heap->tlabs_retire(false); } - if (!heap->collection_set()->is_empty()) { + size_t humongous_regions_promoted = heap->get_promotable_humongous_regions(); + size_t regular_regions_promoted_in_place = heap->get_regular_regions_promoted_in_place(); + if (!heap->collection_set()->is_empty() || (humongous_regions_promoted + regular_regions_promoted_in_place > 0)) { + // Even if the collection set is empty, we need to do evacuation if there are regions to be promoted in place. + // Degenerated evacuation takes responsibility for registering objects and setting the remembered set cards to dirty. + + if (ShenandoahVerify) { + heap->verifier()->verify_before_evacuation(); + } + heap->set_evacuation_in_progress(true); heap->set_has_forwarded_objects(true); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index 3ac7507941132..c873973459704 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -38,7 +38,6 @@ #include "memory/resourceArea.hpp" #include "runtime/orderAccess.hpp" - ShenandoahSetsOfFree::ShenandoahSetsOfFree(size_t max_regions, ShenandoahFreeSet* free_set) : _max(max_regions), _free_set(free_set), @@ -162,9 +161,16 @@ void ShenandoahSetsOfFree::move_to_set(size_t idx, ShenandoahFreeMemoryType new_ // During flip_to_gc: // Mutator empty => Collector // Mutator empty => Old Collector - assert (((region_capacity < _region_size_bytes) && (orig_set == Mutator) && (new_set == Collector)) || - ((region_capacity == _region_size_bytes) && (orig_set == Mutator) && (new_set == Collector || new_set == OldCollector)), - "Unexpected movement between sets"); + // At start of update refs: + // Collector => Mutator + // OldCollector Empty => Mutator + assert (((region_capacity <= _region_size_bytes) && + ((orig_set == Mutator) && (new_set == Collector)) || + ((orig_set == Collector) && (new_set == Mutator))) || + ((region_capacity == _region_size_bytes) && + ((orig_set == Mutator) && (new_set == Collector)) || + ((orig_set == OldCollector) && (new_set == Mutator)) || + (new_set == OldCollector)), "Unexpected movement between sets"); _membership[idx] = new_set; _capacity_of[orig_set] -= region_capacity; @@ -398,7 +404,6 @@ void ShenandoahSetsOfFree::assert_bounds() { } #endif - ShenandoahFreeSet::ShenandoahFreeSet(ShenandoahHeap* heap, size_t max_regions) : _heap(heap), _free_sets(max_regions, this) @@ -412,6 +417,7 @@ ShenandoahFreeSet::ShenandoahFreeSet(ShenandoahHeap* heap, size_t max_regions) : HeapWord* ShenandoahFreeSet::allocate_old_with_affiliation(ShenandoahAffiliation affiliation, ShenandoahAllocRequest& req, bool& in_new_region) { shenandoah_assert_heaplocked(); + size_t rightmost = (affiliation == ShenandoahAffiliation::FREE)? _free_sets.rightmost_empty(OldCollector): _free_sets.rightmost(OldCollector); size_t leftmost = @@ -450,7 +456,19 @@ HeapWord* ShenandoahFreeSet::allocate_old_with_affiliation(ShenandoahAffiliation return nullptr; } -HeapWord* ShenandoahFreeSet::allocate_with_affiliation(ShenandoahAffiliation affiliation, ShenandoahAllocRequest& req, bool& in_new_region) { +void ShenandoahFreeSet::add_old_collector_free_region(ShenandoahHeapRegion* region) { + shenandoah_assert_heaplocked(); + size_t idx = region->index(); + size_t capacity = alloc_capacity(region); + assert(_free_sets.membership(idx) == NotFree, "Regions promoted in place should not be in any free set"); + if (capacity >= PLAB::min_size() * HeapWordSize) { + _free_sets.make_free(idx, OldCollector, capacity); + _heap->augment_promo_reserve(capacity); + } +} + +HeapWord* ShenandoahFreeSet::allocate_with_affiliation(ShenandoahAffiliation affiliation, + ShenandoahAllocRequest& req, bool& in_new_region) { shenandoah_assert_heaplocked(); size_t rightmost = (affiliation == ShenandoahAffiliation::FREE)? _free_sets.rightmost_empty(Collector): _free_sets.rightmost(Collector); @@ -469,7 +487,8 @@ HeapWord* ShenandoahFreeSet::allocate_with_affiliation(ShenandoahAffiliation aff } } } - log_debug(gc, free)("Could not allocate collector region with affiliation: %s for request " PTR_FORMAT, shenandoah_affiliation_name(affiliation), p2i(&req)); + log_debug(gc, free)("Could not allocate collector region with affiliation: %s for request " PTR_FORMAT, + shenandoah_affiliation_name(affiliation), p2i(&req)); return nullptr; } @@ -494,15 +513,15 @@ HeapWord* ShenandoahFreeSet::allocate_single(ShenandoahAllocRequest& req, bool& if (_heap->mode()->is_generational()) { switch (req.affiliation()) { case ShenandoahAffiliation::OLD_GENERATION: - // Note: unsigned result from adjusted_unaffiliated_regions() will never be less than zero, but it may equal zero. - if (_heap->old_generation()->adjusted_unaffiliated_regions() <= 0) { + // Note: unsigned result from free_unaffiliated_regions() will never be less than zero, but it may equal zero. + if (_heap->old_generation()->free_unaffiliated_regions() <= 0) { allow_new_region = false; } break; case ShenandoahAffiliation::YOUNG_GENERATION: - // Note: unsigned result from adjusted_unaffiliated_regions() will never be less than zero, but it may equal zero. - if (_heap->young_generation()->adjusted_unaffiliated_regions() <= 0) { + // Note: unsigned result from free_unaffiliated_regions() will never be less than zero, but it may equal zero. + if (_heap->young_generation()->free_unaffiliated_regions() <= 0) { allow_new_region = false; } break; @@ -515,7 +534,6 @@ HeapWord* ShenandoahFreeSet::allocate_single(ShenandoahAllocRequest& req, bool& break; } } - switch (req.type()) { case ShenandoahAllocRequest::_alloc_tlab: case ShenandoahAllocRequest::_alloc_shared: { @@ -524,8 +542,9 @@ HeapWord* ShenandoahFreeSet::allocate_single(ShenandoahAllocRequest& req, bool& ShenandoahHeapRegion* r = _heap->get_region(idx); if (_free_sets.in_free_set(idx, Mutator) && (allow_new_region || r->is_affiliated())) { // try_allocate_in() increases used if the allocation is successful. - HeapWord* result = try_allocate_in(r, req, in_new_region); - if (result != nullptr) { + HeapWord* result; + size_t min_size = (req.type() == ShenandoahAllocRequest::_alloc_tlab)? req.min_size(): req.size(); + if ((alloc_capacity(r) >= min_size) && ((result = try_allocate_in(r, req, in_new_region)) != nullptr)) { return result; } } @@ -576,19 +595,16 @@ HeapWord* ShenandoahFreeSet::allocate_single(ShenandoahAllocRequest& req, bool& } } } - // No dice. Can we borrow space from mutator view? if (!ShenandoahEvacReserveOverflow) { return nullptr; } - // TODO: - // if (!allow_new_region && req.is_old() && (young_generation->adjusted_unaffiliated_regions() > 0)) { - // transfer a region from young to old; - // allow_new_region = true; - // heap->set_old_evac_reserve(heap->get_old_evac_reserve() + region_size_bytes); - // } - // + if (!allow_new_region && req.is_old() && (_heap->young_generation()->free_unaffiliated_regions() > 0)) { + // This allows us to flip a mutator region to old_collector + allow_new_region = true; + } + // We should expand old-gen if this can prevent an old-gen evacuation failure. We don't care so much about // promotion failures since they can be mitigated in a subsequent GC pass. Would be nice to know if this // allocation request is for evacuation or promotion. Individual threads limit their use of PLAB memory for @@ -706,6 +722,9 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah // coalesce-and-fill processing. r->end_preemptible_coalesce_and_fill(); _heap->clear_cards_for(r); + _heap->old_generation()->increment_affiliated_region_count(); + } else { + _heap->young_generation()->increment_affiliated_region_count(); } assert(ctx->top_at_mark_start(r) == r->bottom(), "Newly established allocation region starts with TAMS equal to bottom"); @@ -727,6 +746,8 @@ HeapWord* ShenandoahFreeSet::try_allocate_in(ShenandoahHeapRegion* r, Shenandoah // req.size() is in words, r->free() is in bytes. if (ShenandoahElasticTLAB && req.is_lab_alloc()) { if (req.type() == ShenandoahAllocRequest::_alloc_plab) { + assert(_heap->mode()->is_generational(), "PLABs are only for generational mode"); + assert(_free_sets.in_free_set(r->index(), OldCollector), "PLABS must be allocated in old_collector_free regions"); // Need to assure that plabs are aligned on multiple of card region. // Since we have Elastic TLABs, align sizes up. They may be decreased to fit in the usable // memory remaining in the region (which will also be aligned to cards). @@ -838,7 +859,7 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req) { // Check if there are enough regions left to satisfy allocation. if (_heap->mode()->is_generational()) { - size_t avail_young_regions = generation->adjusted_unaffiliated_regions(); + size_t avail_young_regions = generation->free_unaffiliated_regions(); if (num > _free_sets.count(Mutator) || (num > avail_young_regions)) { return nullptr; } @@ -908,6 +929,8 @@ HeapWord* ShenandoahFreeSet::allocate_contiguous(ShenandoahAllocRequest& req) { // While individual regions report their true use, all humongous regions are marked used in the free set. _free_sets.remove_from_free_sets(r->index()); } + _heap->young_generation()->increase_affiliated_region_count(num); + size_t total_humongous_size = ShenandoahHeapRegion::region_size_bytes() * num; _free_sets.increase_used(Mutator, total_humongous_size); _free_sets.assert_bounds(); @@ -925,6 +948,11 @@ bool ShenandoahFreeSet::can_allocate_from(ShenandoahHeapRegion *r) const { return r->is_empty() || (r->is_trash() && !_heap->is_concurrent_weak_root_in_progress()); } +bool ShenandoahFreeSet::can_allocate_from(size_t idx) const { + ShenandoahHeapRegion* r = _heap->get_region(idx); + return can_allocate_from(r); +} + size_t ShenandoahFreeSet::alloc_capacity(size_t idx) const { ShenandoahHeapRegion* r = _heap->get_region(idx); return alloc_capacity(r); @@ -976,14 +1004,15 @@ void ShenandoahFreeSet::flip_to_old_gc(ShenandoahHeapRegion* r) { size_t idx = r->index(); assert(_free_sets.in_free_set(idx, Mutator), "Should be in mutator view"); + // Note: can_allocate_from(r) means r is entirely empty assert(can_allocate_from(r), "Should not be allocated"); size_t region_capacity = alloc_capacity(r); _free_sets.move_to_set(idx, OldCollector, region_capacity); _free_sets.assert_bounds(); - - // We do not ensure that the region is no longer trash, - // relying on try_allocate_in(), which always comes next, + _heap->generation_sizer()->force_transfer_to_old(1); + _heap->augment_old_evac_reserve(region_capacity); + // We do not ensure that the region is no longer trash, relying on try_allocate_in(), which always comes next, // to recycle trash before attempting to allocate anything in the region. } @@ -997,8 +1026,7 @@ void ShenandoahFreeSet::flip_to_gc(ShenandoahHeapRegion* r) { _free_sets.move_to_set(idx, Collector, region_capacity); _free_sets.assert_bounds(); - // We do not ensure that the region is no longer trash, - // relying on try_allocate_in(), which always comes next, + // We do not ensure that the region is no longer trash, relying on try_allocate_in(), which always comes next, // to recycle trash before attempting to allocate anything in the region. } @@ -1016,10 +1044,21 @@ void ShenandoahFreeSet::clear_internal() { // move some of the mutator regions into the collector set or old_collector set with the intent of packing // old_collector memory into the highest (rightmost) addresses of the heap and the collector memory into the // next highest addresses of the heap, with mutator memory consuming the lowest addresses of the heap. -void ShenandoahFreeSet::find_regions_with_alloc_capacity() { +void ShenandoahFreeSet::find_regions_with_alloc_capacity(size_t &young_cset_regions, size_t &old_cset_regions) { + old_cset_regions = 0; + young_cset_regions = 0; for (size_t idx = 0; idx < _heap->num_regions(); idx++) { ShenandoahHeapRegion* region = _heap->get_region(idx); + if (region->is_trash()) { + // Trashed regions represent regions that had been in the collection set but have not yet been "cleaned up". + if (region->is_old()) { + old_cset_regions++; + } else { + assert(region->is_young(), "Trashed region should be old or young"); + young_cset_regions++; + } + } if (region->is_alloc_allowed() || region->is_trash()) { assert(!region->is_cset(), "Shouldn't be adding cset regions to the free set"); assert(_free_sets.in_free_set(idx, NotFree), "We are about to make region free; it should not be free already"); @@ -1044,20 +1083,124 @@ void ShenandoahFreeSet::find_regions_with_alloc_capacity() { } } -void ShenandoahFreeSet::rebuild() { +// Move no more than cset_regions from the existing Collector and OldCollector free sets to the Mutator free set. +// This is called from outside the heap lock. +void ShenandoahFreeSet::move_collector_sets_to_mutator(size_t max_xfer_regions) { + size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); + size_t collector_empty_xfer = 0; + size_t collector_not_empty_xfer = 0; + size_t old_collector_empty_xfer = 0; + + // Process empty regions within the Collector free set + if ((max_xfer_regions > 0) && (_free_sets.leftmost_empty(Collector) <= _free_sets.rightmost_empty(Collector))) { + ShenandoahHeapLocker locker(_heap->lock()); + for (size_t idx = _free_sets.leftmost_empty(Collector); + (max_xfer_regions > 0) && (idx <= _free_sets.rightmost_empty(Collector)); idx++) { + if (_free_sets.in_free_set(idx, Collector) && can_allocate_from(idx)) { + _free_sets.move_to_set(idx, Mutator, region_size_bytes); + max_xfer_regions--; + collector_empty_xfer += region_size_bytes; + } + } + } + + // Process empty regions within the OldCollector free set + size_t old_collector_regions = 0; + if ((max_xfer_regions > 0) && (_free_sets.leftmost_empty(OldCollector) <= _free_sets.rightmost_empty(OldCollector))) { + ShenandoahHeapLocker locker(_heap->lock()); + for (size_t idx = _free_sets.leftmost_empty(OldCollector); + (max_xfer_regions > 0) && (idx <= _free_sets.rightmost_empty(OldCollector)); idx++) { + if (_free_sets.in_free_set(idx, OldCollector) && can_allocate_from(idx)) { + _free_sets.move_to_set(idx, Mutator, region_size_bytes); + max_xfer_regions--; + old_collector_empty_xfer += region_size_bytes; + old_collector_regions++; + } + } + if (old_collector_regions > 0) { + _heap->generation_sizer()->transfer_to_young(old_collector_regions); + } + } + + // If there are any non-empty regions within Collector set, we can also move them to the Mutator free set + if ((max_xfer_regions > 0) && (_free_sets.leftmost(Collector) <= _free_sets.rightmost(Collector))) { + ShenandoahHeapLocker locker(_heap->lock()); + for (size_t idx = _free_sets.leftmost(Collector); (max_xfer_regions > 0) && (idx <= _free_sets.rightmost(Collector)); idx++) { + size_t alloc_capacity = this->alloc_capacity(idx); + if (_free_sets.in_free_set(idx, Collector) && (alloc_capacity > 0)) { + _free_sets.move_to_set(idx, Mutator, alloc_capacity); + max_xfer_regions--; + collector_not_empty_xfer += alloc_capacity; + } + } + } + + size_t collector_xfer = collector_empty_xfer + collector_not_empty_xfer; + size_t total_xfer = collector_xfer + old_collector_empty_xfer; + log_info(gc, free)("At start of update refs, moving " SIZE_FORMAT "%s to Mutator free set from Collector Reserve (" + SIZE_FORMAT "%s) and from Old Collector Reserve (" SIZE_FORMAT "%s)", + byte_size_in_proper_unit(total_xfer), proper_unit_for_byte_size(total_xfer), + byte_size_in_proper_unit(collector_xfer), proper_unit_for_byte_size(collector_xfer), + byte_size_in_proper_unit(old_collector_empty_xfer), proper_unit_for_byte_size(old_collector_empty_xfer)); +} + + +// Overwrite arguments to represent the amount of memory in each generation that is about to be recycled +void ShenandoahFreeSet::prepare_to_rebuild(size_t &young_cset_regions, size_t &old_cset_regions) { shenandoah_assert_heaplocked(); // This resets all state information, removing all regions from all sets. clear(); - log_debug(gc, free)("Rebuilding FreeSet"); // This places regions that have alloc_capacity into the old_collector set if they identify as is_old() or the // mutator set otherwise. - find_regions_with_alloc_capacity(); + find_regions_with_alloc_capacity(young_cset_regions, old_cset_regions); +} + +void ShenandoahFreeSet::rebuild(size_t young_cset_regions, size_t old_cset_regions) { + shenandoah_assert_heaplocked(); + size_t young_reserve, old_reserve; + size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); + + size_t old_capacity = _heap->old_generation()->max_capacity(); + size_t old_available = _heap->old_generation()->available(); + size_t old_unaffiliated_regions = _heap->old_generation()->free_unaffiliated_regions(); + size_t young_capacity = _heap->young_generation()->max_capacity(); + size_t young_available = _heap->young_generation()->available(); + size_t young_unaffiliated_regions = _heap->young_generation()->free_unaffiliated_regions(); + + old_unaffiliated_regions += old_cset_regions; + old_available += old_cset_regions * region_size_bytes; + young_unaffiliated_regions += young_cset_regions; + young_available += young_cset_regions * region_size_bytes; + + // Consult old-region surplus and deficit to make adjustments to current generation capacities and availability. + // The generation region transfers take place after we rebuild. + size_t old_region_surplus = _heap->get_old_region_surplus(); + size_t old_region_deficit = _heap->get_old_region_deficit(); + + if (old_region_surplus > 0) { + size_t xfer_bytes = old_region_surplus * region_size_bytes; + assert(old_region_surplus <= old_unaffiliated_regions, "Cannot transfer regions that are affiliated"); + old_capacity -= xfer_bytes; + old_available -= xfer_bytes; + old_unaffiliated_regions -= old_region_surplus; + young_capacity += xfer_bytes; + young_available += xfer_bytes; + young_unaffiliated_regions += old_region_surplus; + } else if (old_region_deficit > 0) { + size_t xfer_bytes = old_region_deficit * region_size_bytes; + assert(old_region_deficit <= young_unaffiliated_regions, "Cannot transfer regions that are affiliated"); + old_capacity += xfer_bytes; + old_available += xfer_bytes; + old_unaffiliated_regions += old_region_deficit; + young_capacity -= xfer_bytes;; + young_available -= xfer_bytes; + young_unaffiliated_regions -= old_region_deficit; + } // Evac reserve: reserve trailing space for evacuations, with regions reserved for old evacuations placed to the right // of regions reserved of young evacuations. - size_t young_reserve, old_reserve; if (!_heap->mode()->is_generational()) { young_reserve = (_heap->max_capacity() / 100) * ShenandoahEvacReserve; old_reserve = 0; @@ -1070,15 +1213,32 @@ void ShenandoahFreeSet::rebuild() { // We are rebuilding at the end of final mark, having already established evacuation budgets for this GC pass. young_reserve = _heap->get_young_evac_reserve(); old_reserve = _heap->get_promoted_reserve() + _heap->get_old_evac_reserve(); + assert(old_reserve <= old_available, + "Cannot reserve (" SIZE_FORMAT " + " SIZE_FORMAT") more OLD than is available: " SIZE_FORMAT, + _heap->get_promoted_reserve(), _heap->get_old_evac_reserve(), old_available); } else { // We are rebuilding at end of GC, so we set aside budgets specified on command line (or defaults) - young_reserve = (_heap->young_generation()->max_capacity() * ShenandoahEvacReserve) / 100; - old_reserve = MAX2((_heap->old_generation()->max_capacity() * ShenandoahOldEvacReserve) / 100, - ShenandoahOldCompactionReserve * ShenandoahHeapRegion::region_size_bytes()); + young_reserve = (young_capacity * ShenandoahEvacReserve) / 100; + // The auto-sizer has already made old-gen large enough to hold all anticipated evacuations and promotions. + // Affiliated old-gen regions are already in the OldCollector free set. Add in the relevant number of + // unaffiliated regions. + old_reserve = old_available; } } - reserve_regions(young_reserve, old_reserve); + if (old_reserve > _free_sets.capacity_of(OldCollector)) { + // Old available regions that have less than PLAB::min_size() of available memory are not placed into the OldCollector + // free set. Because of this, old_available may not have enough memory to represent the intended reserve. Adjust + // the reserve downward to account for this possibility. This loss is part of the reason why the original budget + // was adjusted with ShenandoahOldEvacWaste and ShenandoahOldPromoWaste multipliers. + if (old_reserve > _free_sets.capacity_of(OldCollector) + old_unaffiliated_regions * region_size_bytes) { + old_reserve = _free_sets.capacity_of(OldCollector) + old_unaffiliated_regions * region_size_bytes; + } + } + if (young_reserve > young_unaffiliated_regions * region_size_bytes) { + young_reserve = young_unaffiliated_regions * region_size_bytes; + } + reserve_regions(young_reserve, old_reserve); _free_sets.establish_alloc_bias(OldCollector); _free_sets.assert_bounds(); log_status(); @@ -1090,7 +1250,8 @@ void ShenandoahFreeSet::rebuild() { // the collector set is at least to_reserve, and the memory available for allocations within the old collector set // is at least to_reserve_old. void ShenandoahFreeSet::reserve_regions(size_t to_reserve, size_t to_reserve_old) { - for (size_t idx = _heap->num_regions() - 1; idx > 0; idx--) { + for (size_t i = _heap->num_regions(); i > 0; i--) { + size_t idx = i - 1; ShenandoahHeapRegion* r = _heap->get_region(idx); if (_free_sets.in_free_set(idx, Mutator)) { assert (!r->is_old(), "mutator_is_free regions should not be affiliated OLD"); @@ -1128,6 +1289,17 @@ void ShenandoahFreeSet::log_status() { size_t retired_young = 0; size_t retired_young_humongous = 0; size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); + size_t retired_young_waste = 0; + size_t retired_old_waste = 0; + size_t consumed_collector = 0; + size_t consumed_old_collector = 0; + size_t consumed_mutator = 0; + size_t available_old = 0; + size_t available_young = 0; + size_t available_mutator = 0; + size_t available_collector = 0; + size_t available_old_collector = 0; + char buffer[BUFFER_SIZE]; for (uint i = 0; i < BUFFER_SIZE; i++) { buffer[i] = '\0'; @@ -1151,12 +1323,21 @@ void ShenandoahFreeSet::log_status() { } if (_free_sets.in_free_set(i, Mutator)) { assert(!r->is_old(), "Old regions should not be in mutator_free set"); - buffer[idx] = (alloc_capacity(r) == region_size_bytes)? 'M': 'm'; + size_t capacity = alloc_capacity(r); + available_mutator += capacity; + consumed_mutator += region_size_bytes - capacity; + buffer[idx] = (capacity == region_size_bytes)? 'M': 'm'; } else if (_free_sets.in_free_set(i, Collector)) { assert(!r->is_old(), "Old regions should not be in collector_free set"); - buffer[idx] = (alloc_capacity(r) == region_size_bytes)? 'C': 'c'; + size_t capacity = alloc_capacity(r); + available_collector += capacity; + consumed_collector += region_size_bytes - capacity; + buffer[idx] = (capacity == region_size_bytes)? 'C': 'c'; } else if (_free_sets.in_free_set(i, OldCollector)) { - buffer[idx] = (alloc_capacity(r) == region_size_bytes)? 'O': 'o'; + size_t capacity = alloc_capacity(r); + available_old_collector += capacity; + consumed_old_collector += region_size_bytes - capacity; + buffer[idx] = (capacity == region_size_bytes)? 'O': 'o'; } else if (r->is_humongous()) { if (r->is_old()) { buffer[idx] = 'H'; @@ -1168,9 +1349,11 @@ void ShenandoahFreeSet::log_status() { } else { if (r->is_old()) { buffer[idx] = '~'; + retired_old_waste += alloc_capacity(r); retired_old += region_size_bytes; } else { buffer[idx] = '_'; + retired_young_waste += alloc_capacity(r); retired_young += region_size_bytes; } } @@ -1184,12 +1367,6 @@ void ShenandoahFreeSet::log_status() { log_info(gc, free)(" %6u: %s", (uint) (_heap->num_regions() - remnant), buffer); size_t total_young = retired_young + retired_young_humongous; size_t total_old = retired_old + retired_old_humongous; - log_info(gc, free)("Retired young: " SIZE_FORMAT "%s (including humongous: " SIZE_FORMAT "%s), old: " SIZE_FORMAT - "%s (including humongous: " SIZE_FORMAT "%s)", - byte_size_in_proper_unit(total_young), proper_unit_for_byte_size(total_young), - byte_size_in_proper_unit(retired_young_humongous), proper_unit_for_byte_size(retired_young_humongous), - byte_size_in_proper_unit(total_old), proper_unit_for_byte_size(total_old), - byte_size_in_proper_unit(retired_old_humongous), proper_unit_for_byte_size(retired_old_humongous)); } #endif diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp index ef43c236eab37..bbf6f1cafdac8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.hpp @@ -72,7 +72,7 @@ class ShenandoahSetsOfFree { // Place region idx into free set which_set. Requires that idx is currently NotFree. void make_free(size_t idx, ShenandoahFreeMemoryType which_set, size_t region_capacity); - // Place region idx into free set new_set. Requires that idx is currently not NotFRee. + // Place region idx into free set new_set. Requires that idx is currently not NotFree. void move_to_set(size_t idx, ShenandoahFreeMemoryType new_set, size_t region_capacity); // Returns the ShenandoahFreeMemoryType affiliation of region idx, or NotFree if this region is not currently free. This does @@ -172,11 +172,20 @@ class ShenandoahFreeSet : public CHeapObj { void flip_to_gc(ShenandoahHeapRegion* r); void flip_to_old_gc(ShenandoahHeapRegion* r); + void adjust_bounds_for_additional_old_collector_free_region(size_t idx); + + void recompute_bounds(); + void adjust_bounds(); + bool touches_bounds(size_t num) const; + + // Used of free set represents the amount of is_mutator_free set that has been consumed since most recent rebuild. + void increase_used(size_t amount); void clear_internal(); void try_recycle_trashed(ShenandoahHeapRegion *r); bool can_allocate_from(ShenandoahHeapRegion *r) const; + bool can_allocate_from(size_t idx) const; bool has_alloc_capacity(size_t idx) const; bool has_alloc_capacity(ShenandoahHeapRegion *r) const; bool has_no_alloc_capacity(ShenandoahHeapRegion *r) const; @@ -188,7 +197,11 @@ class ShenandoahFreeSet : public CHeapObj { size_t alloc_capacity(size_t idx) const; void clear(); - void rebuild(); + void prepare_to_rebuild(size_t &young_cset_regions, size_t &old_cset_regions); + void rebuild(size_t young_cset_regions, size_t old_cset_regions); + void move_collector_sets_to_mutator(size_t cset_regions); + + void add_old_collector_free_region(ShenandoahHeapRegion* region); void recycle_trash(); @@ -209,7 +222,7 @@ class ShenandoahFreeSet : public CHeapObj { void print_on(outputStream* out) const; - void find_regions_with_alloc_capacity(); + void find_regions_with_alloc_capacity(size_t &young_cset_regions, size_t &old_cset_regions); void reserve_regions(size_t young_reserve, size_t old_reserve); }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp index 79c8b1c134520..db886618f386a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.cpp @@ -175,14 +175,21 @@ void ShenandoahFullGC::op_full(GCCause::Cause cause) { metrics.snap_after(); if (heap->mode()->is_generational()) { + heap->mmu_tracker()->record_full(heap->global_generation(), GCId::current()); heap->log_heap_status("At end of Full GC"); // Since we allow temporary violation of these constraints during Full GC, we want to enforce that the assertions are // made valid by the time Full GC completes. - assert(heap->old_generation()->used_regions_size() <= heap->old_generation()->adjusted_capacity(), + assert(heap->old_generation()->used_regions_size() <= heap->old_generation()->max_capacity(), "Old generation affiliated regions must be less than capacity"); - assert(heap->young_generation()->used_regions_size() <= heap->young_generation()->adjusted_capacity(), + assert(heap->young_generation()->used_regions_size() <= heap->young_generation()->max_capacity(), "Young generation affiliated regions must be less than capacity"); + + assert((heap->young_generation()->used() + heap->young_generation()->get_humongous_waste()) + <= heap->young_generation()->used_regions_size(), "Young consumed can be no larger than span of affiliated regions"); + assert((heap->old_generation()->used() + heap->old_generation()->get_humongous_waste()) + <= heap->old_generation()->used_regions_size(), "Old consumed can be no larger than span of affiliated regions"); + } if (metrics.is_good_progress()) { ShenandoahHeap::heap()->notify_gc_progress(); @@ -199,11 +206,7 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) { heap->set_gc_generation(heap->global_generation()); if (heap->mode()->is_generational()) { - // Defer unadjust_available() invocations until after Full GC finishes its efforts because Full GC makes use - // of young-gen memory that may have been loaned from old-gen. - // No need for old_gen->increase_used() as this was done when plabs were allocated. - heap->set_alloc_supplement_reserve(0); heap->set_young_evac_reserve(0); heap->set_old_evac_reserve(0); heap->reset_old_evac_expended(); @@ -342,8 +345,6 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) { // Resize metaspace MetaspaceGC::compute_new_size(); - heap->adjust_generation_sizes(); - // Free worker slices for (uint i = 0; i < heap->max_workers(); i++) { delete worker_slices[i]; @@ -357,10 +358,7 @@ void ShenandoahFullGC::do_it(GCCause::Cause gc_cause) { heap->verifier()->verify_after_fullgc(); } - // Having reclaimed all dead memory, it is now safe to restore capacities to original values. - heap->young_generation()->unadjust_available(); - heap->old_generation()->unadjust_available(); - + // Humongous regions are promoted on demand and are accounted for by normal Full GC mechanisms. if (VerifyAfterGC) { Universe::verify(); } @@ -1340,7 +1338,6 @@ class ShenandoahPostCompactClosure : public ShenandoahHeapRegionClosure { account_for_region(r, _young_regions, _young_usage, _young_humongous_waste); } } - r->set_live_data(live); r->reset_alloc_metadata(); } @@ -1501,12 +1498,84 @@ void ShenandoahFullGC::phase5_epilog() { ShenandoahPostCompactClosure post_compact; heap->heap_region_iterate(&post_compact); post_compact.update_generation_usage(); - log_info(gc)("FullGC done: global usage: " SIZE_FORMAT "%s, young usage: " SIZE_FORMAT "%s, old usage: " SIZE_FORMAT "%s", - byte_size_in_proper_unit(heap->global_generation()->used()), proper_unit_for_byte_size(heap->global_generation()->used()), - byte_size_in_proper_unit(heap->young_generation()->used()), proper_unit_for_byte_size(heap->young_generation()->used()), - byte_size_in_proper_unit(heap->old_generation()->used()), proper_unit_for_byte_size(heap->old_generation()->used())); + if (heap->mode()->is_generational()) { + size_t old_usage = heap->old_generation()->used_regions_size(); + size_t old_capacity = heap->old_generation()->max_capacity(); + + assert(old_usage % ShenandoahHeapRegion::region_size_bytes() == 0, "Old usage must aligh with region size"); + assert(old_capacity % ShenandoahHeapRegion::region_size_bytes() == 0, "Old capacity must aligh with region size"); + + if (old_capacity > old_usage) { + size_t excess_old_regions = (old_capacity - old_usage) / ShenandoahHeapRegion::region_size_bytes(); + heap->generation_sizer()->transfer_to_young(excess_old_regions); + } else if (old_capacity < old_usage) { + size_t old_regions_deficit = (old_usage - old_capacity) / ShenandoahHeapRegion::region_size_bytes(); + heap->generation_sizer()->transfer_to_old(old_regions_deficit); + } + + log_info(gc)("FullGC done: young usage: " SIZE_FORMAT "%s, old usage: " SIZE_FORMAT "%s", + byte_size_in_proper_unit(heap->young_generation()->used()), proper_unit_for_byte_size(heap->young_generation()->used()), + byte_size_in_proper_unit(heap->old_generation()->used()), proper_unit_for_byte_size(heap->old_generation()->used())); + } heap->collection_set()->clear(); - heap->free_set()->rebuild(); + size_t young_cset_regions, old_cset_regions; + heap->free_set()->prepare_to_rebuild(young_cset_regions, old_cset_regions); + + // We also do not expand old generation size following Full GC because we have scrambled age populations and + // no longer have objects separated by age into distinct regions. + + // TODO: Do we need to fix FullGC so that it maintains aged segregation of objects into distinct regions? + // A partial solution would be to remember how many objects are of tenure age following Full GC, but + // this is probably suboptimal, because most of these objects will not reside in a region that will be + // selected for the next evacuation phase. + + // In case this Full GC resulted from degeneration, clear the tally on anticipated promotion. + heap->clear_promotion_potential(); + heap->clear_promotion_in_place_potential(); + + if (heap->mode()->is_generational()) { + // Invoke this in case we are able to transfer memory from OLD to YOUNG. + heap->adjust_generation_sizes_for_next_cycle(0, 0, 0); + } + heap->free_set()->rebuild(young_cset_regions, old_cset_regions); + + // We defer generation resizing actions until after cset regions have been recycled. We do this even following an + // abbreviated cycle. + if (heap->mode()->is_generational()) { + bool success; + size_t region_xfer; + const char* region_destination; + ShenandoahYoungGeneration* young_gen = heap->young_generation(); + ShenandoahGeneration* old_gen = heap->old_generation(); + + size_t old_region_surplus = heap->get_old_region_surplus(); + size_t old_region_deficit = heap->get_old_region_deficit(); + if (old_region_surplus) { + success = heap->generation_sizer()->transfer_to_young(old_region_surplus); + region_destination = "young"; + region_xfer = old_region_surplus; + } else if (old_region_deficit) { + success = heap->generation_sizer()->transfer_to_old(old_region_deficit); + region_destination = "old"; + region_xfer = old_region_deficit; + if (!success) { + ((ShenandoahOldHeuristics *) old_gen->heuristics())->trigger_cannot_expand(); + } + } else { + region_destination = "none"; + region_xfer = 0; + success = true; + } + heap->set_old_region_surplus(0); + heap->set_old_region_deficit(0); + size_t young_available = young_gen->available(); + size_t old_available = old_gen->available(); + log_info(gc, ergo)("After cleanup, %s " SIZE_FORMAT " regions to %s to prepare for next gc, old available: " + SIZE_FORMAT "%s, young_available: " SIZE_FORMAT "%s", + success? "successfully transferred": "failed to transfer", region_xfer, region_destination, + byte_size_in_proper_unit(old_available), proper_unit_for_byte_size(old_available), + byte_size_in_proper_unit(young_available), proper_unit_for_byte_size(young_available)); + } + heap->clear_cancelled_gc(true /* clear oom handler */); } - heap->clear_cancelled_gc(true /* clear oom handler */); } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index 22281b3250e25..3606e2cf63216 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -225,7 +225,6 @@ void ShenandoahGeneration::prepare_gc() { void ShenandoahGeneration::compute_evacuation_budgets(ShenandoahHeap* heap, bool* preselected_regions, ShenandoahCollectionSet* collection_set, size_t &consumed_by_advance_promotion) { - assert(heap->mode()->is_generational(), "Only generational mode uses evacuation budgets."); size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); size_t regions_available_to_loan = 0; size_t minimum_evacuation_reserve = ShenandoahOldCompactionReserve * region_size_bytes; @@ -249,160 +248,90 @@ void ShenandoahGeneration::compute_evacuation_budgets(ShenandoahHeap* heap, bool // Do not fill up old-gen memory with promotions. Reserve some amount of memory for compaction purposes. size_t young_evac_reserve_max = 0; - if (heap->doing_mixed_evacuations()) { - // Compute old_evacuation_reserve: how much memory are we reserving to hold the results of - // evacuating old-gen heap regions? In order to sustain a consistent pace of young-gen collections, - // the goal is to maintain a consistent value for this parameter (when the candidate set is not - // empty). This value is the minimum of: - // 1. old_gen->available() - // 2. old-gen->capacity() * ShenandoahOldEvacReserve) / 100 - // (e.g. old evacuation should be no larger than 5% of old_gen capacity) - // 3. ((young_gen->capacity * ShenandoahEvacReserve / 100) * ShenandoahOldEvacRatioPercent) / 100 - // (e.g. old evacuation should be no larger than 12% of young-gen evacuation) - old_evacuation_reserve = old_generation->available(); - // This assertion has been disabled because we expect this code to be replaced by 05/2023 - // assert(old_evacuation_reserve > minimum_evacuation_reserve, "Old-gen available has not been preserved!"); - size_t old_evac_reserve_max = old_generation->max_capacity() * ShenandoahOldEvacReserve / 100; - if (old_evac_reserve_max < old_evacuation_reserve) { - old_evacuation_reserve = old_evac_reserve_max; - } - young_evac_reserve_max = - (((young_generation->max_capacity() * ShenandoahEvacReserve) / 100) * ShenandoahOldEvacRatioPercent) / 100; - if (young_evac_reserve_max < old_evacuation_reserve) { - old_evacuation_reserve = young_evac_reserve_max; - } - } - if (minimum_evacuation_reserve > old_generation->available()) { - // Due to round-off errors during enforcement of minimum_evacuation_reserve during previous GC passes, - // there can be slight discrepancies here. - minimum_evacuation_reserve = old_generation->available(); - } + // First priority is to reclaim the easy garbage out of young-gen. - heap->set_old_evac_reserve(old_evacuation_reserve); - heap->reset_old_evac_expended(); - - // Compute the young evacuation reserve: This is how much memory is available for evacuating young-gen objects. - // We ignore the possible effect of promotions, which reduce demand for young-gen evacuation memory. - // - // TODO: We could give special treatment to the regions that have reached promotion age, because we know their - // live data is entirely eligible for promotion. This knowledge can feed both into calculations of young-gen - // evacuation reserve and promotion reserve. - // - // young_evacuation_reserve for young generation: how much memory are we reserving to hold the results - // of evacuating young collection set regions? This is typically smaller than the total amount - // of available memory, and is also smaller than the total amount of marked live memory within - // young-gen. This value is the smaller of - // - // 1. (young_gen->capacity() * ShenandoahEvacReserve) / 100 - // 2. (young_gen->available() + old_gen_memory_available_to_be_loaned - // - // ShenandoahEvacReserve represents the configured target size of the evacuation region. We can only honor - // this target if there is memory available to hold the evacuations. Memory is available if it is already - // free within young gen, or if it can be borrowed from old gen. Since we have not yet chosen the collection - // sets, we do not yet know the exact accounting of how many regions will be freed by this collection pass. - // What we do know is that there will be at least one evacuated young-gen region for each old-gen region that - // is loaned to the evacuation effort (because regions to be collected consume more memory than the compacted - // regions that will replace them). In summary, if there are old-gen regions that are available to hold the - // results of young-gen evacuations, it is safe to loan them for this purpose. At this point, we have not yet - // established a promoted_reserve. We'll do that after we choose the collection set and analyze its impact - // on available memory. - // - // We do not know the evacuation_supplement until after we have computed the collection set. It is not always - // the case that young-regions inserted into the collection set will result in net decrease of in-use regions - // because ShenandoahEvacWaste times multiplied by memory within the region may be larger than the region size. - // The problem is especially relevant to regions that have been inserted into the collection set because they have - // reached tenure age. These regions tend to have much higher utilization (e.g. 95%). These regions also offer - // a unique opportunity because we know that every live object contained within the region is elgible to be - // promoted. Thus, the following implementation treats these regions specially: - // - // 1. Before beginning collection set selection, we tally the total amount of live memory held within regions - // that are known to have reached tenure age. If this memory times ShenandoahEvacWaste is available within - // old-gen memory, establish an advance promotion reserve to hold all or some percentage of these objects. - // This advance promotion reserve is excluded from memory available for holding old-gen evacuations and cannot - // be "loaned" to young gen. - // - // 2. Tenure-aged regions are included in the collection set iff their evacuation size * ShenandoahEvacWaste fits - // within the advance promotion reserve. It is counter productive to evacuate these regions if they cannot be - // evacuated directly into old-gen memory. So if there is not sufficient memory to hold copies of their - // live data right now, we'll just let these regions remain in young for now, to be evacuated by a subsequent - // evacuation pass. - // - // 3. Next, we calculate a young-gen evacuation budget, which is the smaller of the two quantities mentioned - // above. old_gen_memory_available_to_be_loaned is calculated as: - // old_gen->available - (advance-promotion-reserve + old-gen_evacuation_reserve) - // - // 4. When choosing the collection set, special care is taken to assure that the amount of loaned memory required to - // hold the results of evacuation is smaller than the total memory occupied by the regions added to the collection - // set. We need to take these precautions because we do not know how much memory will be reclaimed by evacuation - // until after the collection set has been constructed. The algorithm is as follows: - // - // a. We feed into the algorithm (i) young available at the start of evacuation and (ii) the amount of memory - // loaned from old-gen that is available to hold the results of evacuation. - // b. As candidate regions are added into the young-gen collection set, we maintain accumulations of the amount - // of memory spanned by the collection set regions and the amount of memory that must be reserved to hold - // evacuation results (by multiplying live-data size by ShenandoahEvacWaste). We process candidate regions - // in order of decreasing amounts of garbage. We skip over (and do not include into the collection set) any - // regions that do not satisfy all of the following conditions: - // - // i. The amount of live data within the region as scaled by ShenandoahEvacWaste must fit within the - // relevant evacuation reserve (live data of old-gen regions must fit within the old-evac-reserve, live - // data of young-gen tenure-aged regions must fit within the advance promotion reserve, live data within - // other young-gen regions must fit within the youn-gen evacuation reserve). - // ii. The accumulation of memory consumed by evacuation must not exceed the accumulation of memory reclaimed - // through evacuation by more than young-gen available. - // iii. Other conditions may be enforced as appropriate for specific heuristics. - // - // Note that regions are considered for inclusion in the selection set in order of decreasing amounts of garbage. - // It is possible that a region with a larger amount of garbage will be rejected because it also has a larger - // amount of live data and some region that follows this region in candidate order is included in the collection - // set (because it has less live data and thus can fit within the evacuation limits even though it has less - // garbage). - - size_t young_evacuation_reserve = (young_generation->max_capacity() * ShenandoahEvacReserve) / 100; - // old evacuation can pack into existing partially used regions. young evacuation and loans for young allocations - // need to target regions that do not already hold any old-gen objects. Round down. - regions_available_to_loan = old_generation->free_unaffiliated_regions(); - - size_t required_evacuation_reserve; - // Memory evacuated from old-gen on this pass will be available to hold old-gen evacuations in next pass. - if (old_evacuation_reserve > minimum_evacuation_reserve) { - required_evacuation_reserve = 0; + // maximum_young_evacuation_reserve is upper bound on memory to be evacuated out of young + size_t maximum_young_evacuation_reserve = (young_generation->max_capacity() * ShenandoahEvacReserve) / 100; + size_t young_evacuation_reserve = maximum_young_evacuation_reserve; + size_t excess_young; + if (young_generation->available() > young_evacuation_reserve) { + excess_young = young_generation->available() - young_evacuation_reserve; } else { - required_evacuation_reserve = minimum_evacuation_reserve - old_evacuation_reserve; + young_evacuation_reserve = young_generation->available(); + excess_young = 0; } - - consumed_by_advance_promotion = _heuristics->select_aged_regions( - old_generation->available() - old_evacuation_reserve - required_evacuation_reserve, num_regions, preselected_regions); - size_t net_available_old_regions = - (old_generation->available() - old_evacuation_reserve - consumed_by_advance_promotion) / region_size_bytes; - - if (regions_available_to_loan > net_available_old_regions) { - regions_available_to_loan = net_available_old_regions; + size_t unaffiliated_young = young_generation->free_unaffiliated_regions() * region_size_bytes; + if (excess_young > unaffiliated_young) { + excess_young = unaffiliated_young; + } else { + // round down to multiple of region size + excess_young /= region_size_bytes; + excess_young *= region_size_bytes; + } + // excess_young is available to be transferred to OLD. Assume that OLD will not request any more than had + // already been set aside for its promotion and evacuation needs at the end of previous GC. No need to + // hold back memory for allocation runway. + + ShenandoahOldHeuristics* old_heuristics = heap->old_heuristics(); + + // maximum_old_evacuation_reserve is an upper bound on memory evacuated from old and evacuated to old (promoted). + size_t maximum_old_evacuation_reserve = + maximum_young_evacuation_reserve * ShenandoahOldEvacRatioPercent / (100 - ShenandoahOldEvacRatioPercent); + // Here's the algebra: + // TotalEvacuation = OldEvacuation + YoungEvacuation + // OldEvacuation = TotalEvacuation * (ShenandoahOldEvacRatioPercent/100) + // OldEvacuation = YoungEvacuation * (ShenandoahOldEvacRatioPercent/100)/(1 - ShenandoahOldEvacRatioPercent/100) + // OldEvacuation = YoungEvacuation * ShenandoahOldEvacRatioPercent/(100 - ShenandoahOldEvacRatioPercent) + + if (maximum_old_evacuation_reserve > old_generation->available()) { + maximum_old_evacuation_reserve = old_generation->available(); + } + + // Second priority is to reclaim garbage out of old-gen if there are old-gen collection candidates. Third priority + // is to promote as much as we have room to promote. However, if old-gen memory is in short supply, this means young + // GC is operating under "duress" and was unable to transfer the memory that we would normally expect. In this case, + // old-gen will refrain from compacting itself in order to allow a quicker young-gen cycle (by avoiding the update-refs + // through ALL of old-gen). If there is some memory available in old-gen, we will use this for promotions as promotions + // do not add to the update-refs burden of GC. + + size_t old_promo_reserve; + if (old_heuristics->unprocessed_old_collection_candidates() > 0) { + // We reserved all old-gen memory at end of previous GC to hold anticipated evacuations to old-gen. If this is + // mixed evacuation, reserve all of this memory for compaction of old-gen and do not promote. Prioritize compaction + // over promotion in order to defragment OLD so that it will be better prepared to efficiently receive promoted memory. + old_evacuation_reserve = maximum_old_evacuation_reserve; + old_promo_reserve = 0; + } else { + // Make all old-evacuation memory for promotion, but if we can't use it all for promotion, we'll allow some evacuation. + old_evacuation_reserve = 0; + old_promo_reserve = maximum_old_evacuation_reserve; } - // Otherwise, regions_available_to_loan is less than net_available_old_regions because available memory is - // scattered between multiple partially used regions. + // We see too many old-evacuation failures if we force ourselves to evacuate into regions that are not initially empty. + // So we limit the old-evacuation reserve to unfragmented memory. Even so, old-evacuation is free to fill in nooks and + // crannies within existing partially used regions and it generally tries to do so. + size_t old_free_regions = old_generation->free_unaffiliated_regions(); + size_t old_free_unfragmented = old_free_regions * region_size_bytes; + if (old_evacuation_reserve > old_free_unfragmented) { + size_t delta = old_evacuation_reserve - old_free_unfragmented; + old_evacuation_reserve -= delta; - if (young_evacuation_reserve > young_generation->available()) { - size_t short_fall = young_evacuation_reserve - young_generation->available(); - if (regions_available_to_loan * region_size_bytes >= short_fall) { - old_regions_loaned_for_young_evac = (short_fall + region_size_bytes - 1) / region_size_bytes; - regions_available_to_loan -= old_regions_loaned_for_young_evac; - } else { - old_regions_loaned_for_young_evac = regions_available_to_loan; - regions_available_to_loan = 0; - young_evacuation_reserve = young_generation->available() + old_regions_loaned_for_young_evac * region_size_bytes; - // In this case, there's no memory available for new allocations while evacuating and updating, unless we - // find more old-gen memory to borrow below. - } + // Let promo consume fragments of old-gen memory. + old_promo_reserve += delta; } - // In generational mode, we may end up choosing a young collection set that contains so many promotable objects - // that there is not sufficient space in old generation to hold the promoted objects. That is ok because we have - // assured there is sufficient space in young generation to hold the rejected promotion candidates. These rejected - // promotion candidates will presumably be promoted in a future evacuation cycle. - heap->set_young_evac_reserve(young_evacuation_reserve); collection_set->establish_preselected(preselected_regions); + consumed_by_advance_promotion = _heuristics->select_aged_regions(old_promo_reserve, num_regions, preselected_regions); + assert(consumed_by_advance_promotion <= maximum_old_evacuation_reserve, "Cannot promote more than available old-gen memory"); + if (consumed_by_advance_promotion < old_promo_reserve) { + // If we're in a global collection, this memory can be used for old evacuations + old_evacuation_reserve += old_promo_reserve - consumed_by_advance_promotion; + } + heap->set_young_evac_reserve(young_evacuation_reserve); + heap->set_old_evac_reserve(old_evacuation_reserve); + heap->set_promoted_reserve(consumed_by_advance_promotion); + + // There is no need to expand OLD because all memory used here was set aside at end of previous GC } // Having chosen the collection set, adjust the budgets for generational mode based on its composition. Note @@ -425,356 +354,92 @@ void ShenandoahGeneration::adjust_evacuation_budgets(ShenandoahHeap* heap, Shena // available that results from a decrease in memory consumed by old evacuation is not necessarily available to be loaned // to young-gen. - assert(heap->mode()->is_generational(), "Only generational mode uses evacuation budgets."); - size_t old_regions_loaned_for_young_evac, regions_available_to_loan; size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); ShenandoahOldGeneration* old_generation = heap->old_generation(); ShenandoahYoungGeneration* young_generation = heap->young_generation(); - size_t old_evacuated = collection_set->get_old_bytes_reserved_for_evacuation(); - size_t old_evacuated_committed = (size_t) (ShenandoahEvacWaste * old_evacuated); - size_t old_evacuation_reserve = heap->get_old_evac_reserve(); - // Immediate garbage found during choose_collection_set() is all young - size_t immediate_garbage = collection_set->get_immediate_trash(); - size_t old_available = old_generation->available(); - size_t young_available = young_generation->available() + immediate_garbage; - size_t loaned_regions = 0; - size_t available_loan_remnant = 0; // loaned memory that is not yet dedicated to any particular budget - - // We expect this code to be replaced by 05/01/23. - // - // assert(((consumed_by_advance_promotion * 33) / 32) >= collection_set->get_young_bytes_to_be_promoted() * ShenandoahEvacWaste, - // "Advance promotion (" SIZE_FORMAT ") should be at least young_bytes_to_be_promoted (" SIZE_FORMAT - // ")* ShenandoahEvacWaste, totalling: " SIZE_FORMAT ", within round-off errors of up to 3.125%%", - // consumed_by_advance_promotion, collection_set->get_young_bytes_to_be_promoted(), - // (size_t) (collection_set->get_young_bytes_to_be_promoted() * ShenandoahEvacWaste)); - - // assert(consumed_by_advance_promotion <= (collection_set->get_young_bytes_to_be_promoted() * ShenandoahEvacWaste * 33) / 32, - // "Round-off errors should be less than 3.125%%, consumed by advance: " SIZE_FORMAT ", promoted: " SIZE_FORMAT, - // consumed_by_advance_promotion, (size_t) (collection_set->get_young_bytes_to_be_promoted() * ShenandoahEvacWaste)); + // Preselected regions have been inserted into the collection set, so we no longer need the preselected array. collection_set->abandon_preselected(); + size_t old_evacuated = collection_set->get_old_bytes_reserved_for_evacuation(); + size_t old_evacuated_committed = (size_t) (ShenandoahOldEvacWaste * old_evacuated); + size_t old_evacuation_reserve = heap->get_old_evac_reserve(); + if (old_evacuated_committed > old_evacuation_reserve) { - // This should only happen due to round-off errors when enforcing ShenandoahEvacWaste + // This should only happen due to round-off errors when enforcing ShenandoahOldEvacWaste assert(old_evacuated_committed <= (33 * old_evacuation_reserve) / 32, "Round-off errors should be less than 3.125%%, committed: " SIZE_FORMAT ", reserved: " SIZE_FORMAT, old_evacuated_committed, old_evacuation_reserve); old_evacuated_committed = old_evacuation_reserve; + // Leave old_evac_reserve as previously configured } else if (old_evacuated_committed < old_evacuation_reserve) { - // This may happen if the old-gen collection consumes less than full budget. + // This happens if the old-gen collection consumes less than full budget. old_evacuation_reserve = old_evacuated_committed; heap->set_old_evac_reserve(old_evacuation_reserve); } - // Recompute old_regions_loaned_for_young_evac because young-gen collection set may not need all the memory - // originally reserved. - size_t young_promoted = collection_set->get_young_bytes_to_be_promoted(); - size_t young_promoted_reserve_used = (size_t) (ShenandoahEvacWaste * young_promoted); + size_t young_advance_promoted = collection_set->get_young_bytes_to_be_promoted(); + size_t young_advance_promoted_reserve_used = (size_t) (ShenandoahPromoEvacWaste * young_advance_promoted); size_t young_evacuated = collection_set->get_young_bytes_reserved_for_evacuation(); size_t young_evacuated_reserve_used = (size_t) (ShenandoahEvacWaste * young_evacuated); - // We'll invoke heap->set_young_evac_reserve() further below, after we make additional adjustments to its value - - // Adjust old_regions_loaned_for_young_evac to feed into calculations of promoted_reserve - if (young_evacuated_reserve_used > young_available) { - size_t short_fall = young_evacuated_reserve_used - young_available; - - // region_size_bytes is a power of 2. loan an integral number of regions. - size_t revised_loan_for_young_evacuation = (short_fall + region_size_bytes - 1) / region_size_bytes; - - // available_loan_remnant represents memory loaned from old-gen but not required for young evacuation. - // This is the excess loaned memory that results from rounding the required loan up to an integral number - // of heap regions. This will be dedicated to alloc_supplement below. - available_loan_remnant = (revised_loan_for_young_evacuation * region_size_bytes) - short_fall; - - // We previously loaned more than was required by young-gen evacuation. So claw some of this memory back. - old_regions_loaned_for_young_evac = revised_loan_for_young_evacuation; - loaned_regions = old_regions_loaned_for_young_evac; - } else { - // Undo the previous loan, if any. - old_regions_loaned_for_young_evac = 0; - loaned_regions = 0; - } - - size_t old_bytes_loaned_for_young_evac = old_regions_loaned_for_young_evac * region_size_bytes - available_loan_remnant; - - // Recompute regions_available_to_loan based on possible changes to old_regions_loaned_for_young_evac and - // old_evacuation_reserve. - - // Any decrease in old_regions_loaned_for_young_evac are immediately available to be loaned - // However, a change to old_evacuation_reserve() is not necessarily available to loan, because this memory may - // reside within many fragments scattered throughout old-gen. - - regions_available_to_loan = old_generation->free_unaffiliated_regions(); - size_t working_old_available = old_generation->available(); - - assert(regions_available_to_loan * region_size_bytes <= working_old_available, - "Regions available to loan must be less than available memory"); - - // fragmented_old_total is the amount of memory in old-gen beyond regions_available_to_loan that is otherwise not - // yet dedicated to a particular budget. This memory can be used for promotion_reserve. - size_t fragmented_old_total = working_old_available - regions_available_to_loan * region_size_bytes; - - // fragmented_old_usage is the memory that is dedicated to holding evacuated old-gen objects, which does not need - // to be an integral number of regions. - size_t fragmented_old_usage = old_evacuated_committed + consumed_by_advance_promotion; - - if (fragmented_old_total >= fragmented_old_usage) { - // Seems this will be rare. In this case, all of the memory required for old-gen evacuations and promotions can be - // taken from the existing fragments within old-gen. Reduce this fragmented total by this amount. - fragmented_old_total -= fragmented_old_usage; - // And reduce regions_available_to_loan by the regions dedicated to young_evac. - regions_available_to_loan -= old_regions_loaned_for_young_evac; - } else { - // In this case, we need to dedicate some of the regions_available_to_loan to hold the results of old-gen evacuations - // and promotions. - - size_t unaffiliated_memory_required_for_old = fragmented_old_usage - fragmented_old_total; - size_t unaffiliated_regions_used_by_old = (unaffiliated_memory_required_for_old + region_size_bytes - 1) / region_size_bytes; - regions_available_to_loan -= (unaffiliated_regions_used_by_old + old_regions_loaned_for_young_evac); - - size_t memory_for_promotions_and_old_evac = fragmented_old_total + unaffiliated_regions_used_by_old; - size_t memory_required_for_promotions_and_old_evac = fragmented_old_usage; - size_t excess_fragmented = memory_for_promotions_and_old_evac - memory_required_for_promotions_and_old_evac; - fragmented_old_total = excess_fragmented; - } - - // Subtract from working_old_available old_evacuated_committed and consumed_by_advance_promotion - working_old_available -= fragmented_old_usage; - // And also subtract out the regions loaned for young evacuation - working_old_available -= old_regions_loaned_for_young_evac * region_size_bytes; - - // Assure that old_evacuated_committed + old_bytes_loaned_for_young_evac >= the minimum evacuation reserve - // in order to prevent promotion reserve from violating minimum evacuation reserve. - size_t old_regions_reserved_for_alloc_supplement = 0; - size_t old_bytes_reserved_for_alloc_supplement = 0; - size_t reserved_bytes_for_future_old_evac = 0; - - old_bytes_reserved_for_alloc_supplement = available_loan_remnant; - available_loan_remnant = 0; - - // Memory that has been loaned for young evacuations and old-gen regions in the current mixed-evacuation collection - // set will be available to hold future old-gen evacuations. If this memory is less than the desired amount of memory - // set aside for old-gen compaction reserve, try to set aside additional memory so that it will be available during - // the next mixed evacuation cycle. Note that memory loaned to young-gen for allocation supplement is excluded from - // the old-gen promotion reserve. - size_t future_evac_reserve_regions = old_regions_loaned_for_young_evac + collection_set->get_old_region_count(); - size_t collected_regions = collection_set->get_young_region_count(); - - if (future_evac_reserve_regions < ShenandoahOldCompactionReserve) { - // Require that we loan more memory for holding young evacuations to assure that we have adequate reserves to receive - // old-gen evacuations during subsequent collections. Loaning this memory for an allocation supplement does not - // satisfy our needs because newly allocated objects are not necessarily counter-balanced by reclaimed collection - // set regions. - - // Put this memory into reserve by identifying it as old_regions_loaned_for_young_evac - size_t additional_regions_to_loan = ShenandoahOldCompactionReserve - future_evac_reserve_regions; - - // We can loan additional regions to be repaid from the anticipated recycling of young collection set regions - // provided that these regions are currently available within old-gen memory. - size_t collected_regions_to_loan; - if (collected_regions >= additional_regions_to_loan) { - collected_regions_to_loan = additional_regions_to_loan; - additional_regions_to_loan = 0; - } else if (collected_regions > 0) { - collected_regions_to_loan = collected_regions; - additional_regions_to_loan -= collected_regions_to_loan; - } else { - collected_regions_to_loan = 0; - } - - if (collected_regions_to_loan > 0) { - // We're evacuating at least this many regions, it's ok to use these regions for allocation supplement since - // we'll be able to repay the loan at end of this GC pass, assuming the regions are available. - if (collected_regions_to_loan > regions_available_to_loan) { - collected_regions_to_loan = regions_available_to_loan; - } - old_bytes_reserved_for_alloc_supplement += collected_regions_to_loan * region_size_bytes; - regions_available_to_loan -= collected_regions_to_loan; - loaned_regions += collected_regions_to_loan; - working_old_available -= collected_regions_to_loan * region_size_bytes; - } - - // If there's still memory that we want to exclude from the current promotion reserve, but we are unable to loan - // this memory because fully empty old-gen regions are not available, decrement the working_old_available to make - // sure that this memory is not used to hold the results of old-gen evacuation. - if (additional_regions_to_loan > regions_available_to_loan) { - size_t unloaned_regions = additional_regions_to_loan - regions_available_to_loan; - size_t unloaned_bytes = unloaned_regions * region_size_bytes; - - if (working_old_available < unloaned_bytes) { - // We're in dire straits. We won't be able to reserve all the memory that we want to make available for the - // next old-gen evacuation. We'll reserve as much of it as possible. Setting working_old_available to zero - // means there will be no promotion except for the advance promotion. Note that if some advance promotion fails, - // the object will be evacuated to young-gen so we should still end up reclaiming the entire advance promotion - // collection set. - reserved_bytes_for_future_old_evac = working_old_available; - working_old_available = 0; - } else { - reserved_bytes_for_future_old_evac = unloaned_bytes; - working_old_available -= unloaned_bytes; - } - size_t regions_reserved_for_future_old_evac = - (reserved_bytes_for_future_old_evac + region_size_bytes - 1) / region_size_bytes; - - if (regions_reserved_for_future_old_evac < regions_available_to_loan) { - regions_available_to_loan -= regions_reserved_for_future_old_evac; - } else { - regions_available_to_loan = 0; - } + assert(young_evacuated_reserve_used <= young_generation->available(), "Cannot evacuate more than is available in young"); + heap->set_young_evac_reserve(young_evacuated_reserve_used); - // Since we're in dire straits, zero out fragmented_old_total so this won't be used for promotion; - if (working_old_available > fragmented_old_total) { - working_old_available -= fragmented_old_total; + size_t old_available = old_generation->available(); + // Now that we've established the collection set, we know how much memory is really required by old-gen for evacuation + // and promotion reserves. Try shrinking OLD now in case that gives us a bit more runway for mutator allocations during + // evac and update phases. + size_t old_consumed = old_evacuated_committed + young_advance_promoted_reserve_used; + assert(old_available >= old_consumed, "Cannot consume more than is available"); + size_t excess_old = old_available - old_consumed; + size_t unaffiliated_old_regions = old_generation->free_unaffiliated_regions(); + size_t unaffiliated_old = unaffiliated_old_regions * region_size_bytes; + assert(old_available >= unaffiliated_old, "Unaffiliated old is a subset of old available"); + + // Make sure old_evac_committed is unaffiliated + if (old_evacuated_committed > 0) { + if (unaffiliated_old > old_evacuated_committed) { + size_t giveaway = unaffiliated_old - old_evacuated_committed; + size_t giveaway_regions = giveaway / region_size_bytes; // round down + if (giveaway_regions > 0) { + excess_old = MIN2(excess_old, giveaway_regions * region_size_bytes); } else { - working_old_available = 0; + excess_old = 0; } - fragmented_old_total = 0; + } else { + excess_old = 0; } } - // Establish young_evac_reserve so that this young-gen memory is not used for new allocations, allowing the memory - // to be returned to old-gen as soon as the current collection set regions are reclaimed. - heap->set_young_evac_reserve(young_evacuated_reserve_used); - - // Limit promoted_reserve so that we can set aside memory to be loaned from old-gen to young-gen. This - // value is not "critical". If we underestimate, certain promotions will simply be deferred. If we put - // "all the rest" of old-gen memory into the promotion reserve, we'll have nothing left to loan to young-gen - // during the evac and update phases of GC. So we "limit" the sizes of the promotion budget to be the smaller of: - // - // 1. old_available - // (old_available is old_gen->available() - - // (old_evacuated_committed + consumed_by_advance_promotion + loaned_for_young_evac + reserved_for_alloc_supplement)) - // 2. young bytes reserved for evacuation (we can't promote more than young is evacuating) - size_t promotion_reserve = working_old_available; - - // We experimented with constraining promoted_reserve to be no larger than 4 times the size of previously_promoted, - // but this constraint was too limiting, resulting in failure of legitimate promotions. This was tried before we - // had special handling in place for advance promotion. We should retry now that advance promotion is handled - // specially. - - // We had also experimented with constraining promoted_reserve to be no more than young_evacuation_committed - // divided by promotion_divisor, where: - // size_t promotion_divisor = (0x02 << InitialTenuringThreshold) - 1; - // This also was found to be too limiting, resulting in failure of legitimate promotions. - // - // Both experiments were conducted in the presence of other bugs which could have been the root cause for - // the failures identified above as being "too limiting". TODO: conduct new experiments with the more limiting - // values of young_evacuation_reserved_used. - - // young_evacuation_reserve_used already excludes bytes known to be promoted, which equals consumed_by_advance_promotion - if (young_evacuated_reserve_used < promotion_reserve) { - // Shrink promotion_reserve if it is larger than the memory to be consumed by evacuating all young objects in - // collection set, including anticipated waste. There's no benefit in using a larger promotion_reserve. - // young_evacuation_reserve_used does not include live memory within tenure-aged regions. - promotion_reserve = young_evacuated_reserve_used; - } - assert(working_old_available >= promotion_reserve, "Cannot reserve for promotion more than is available"); - working_old_available -= promotion_reserve; - // Having reserved this memory for promotion, the regions are no longer available to be loaned. - size_t regions_consumed_by_promotion_reserve = (promotion_reserve + region_size_bytes - 1) / region_size_bytes; - if (regions_consumed_by_promotion_reserve > regions_available_to_loan) { - // This can happen if the promotion reserve makes use of memory that is fragmented between many partially available - // old-gen regions. - regions_available_to_loan = 0; - } else { - regions_available_to_loan -= regions_consumed_by_promotion_reserve; + // If we find that OLD has excess regions, give them back to YOUNG now to reduce likelihood we run out of allocation + // runway during evacuation and update-refs. + size_t regions_to_xfer = 0; + if (excess_old > unaffiliated_old) { + // we can give back unaffiliated_old (all of unaffiliated is excess) + if (unaffiliated_old_regions > 0) { + regions_to_xfer = unaffiliated_old_regions; + } + } else if (unaffiliated_old_regions > 0) { + // excess_old < unaffiliated old: we can give back MIN(excess_old/region_size_bytes, unaffiliated_old_regions) + size_t excess_regions = excess_old / region_size_bytes; + size_t regions_to_xfer = MIN2(excess_regions, unaffiliated_old_regions); } - log_debug(gc)("old_gen->available(): " SIZE_FORMAT " divided between promotion reserve: " SIZE_FORMAT - ", old evacuation reserve: " SIZE_FORMAT ", advance promotion reserve supplement: " SIZE_FORMAT - ", old loaned for young evacuation: " SIZE_FORMAT ", old reserved for alloc supplement: " SIZE_FORMAT, - old_generation->available(), promotion_reserve, old_evacuated_committed, consumed_by_advance_promotion, - old_regions_loaned_for_young_evac * region_size_bytes, old_bytes_reserved_for_alloc_supplement); - - promotion_reserve += consumed_by_advance_promotion; - heap->set_promoted_reserve(promotion_reserve); - - heap->reset_promoted_expended(); - if (collection_set->get_old_bytes_reserved_for_evacuation() == 0) { - // Setting old evacuation reserve to zero denotes that there is no old-gen evacuation in this pass. - heap->set_old_evac_reserve(0); + if (regions_to_xfer > 0) { + bool result = heap->generation_sizer()->transfer_to_young(regions_to_xfer); + assert(excess_old > regions_to_xfer * region_size_bytes, "Cannot xfer more than excess old"); + excess_old -= regions_to_xfer * region_size_bytes; + log_info(gc, ergo)("%s transferred " SIZE_FORMAT " excess regions to young before start of evacuation", + result? "Successfully": "Unsuccessfully", regions_to_xfer); } - size_t old_gen_usage_base = old_generation->used() - collection_set->get_old_garbage(); - heap->capture_old_usage(old_gen_usage_base); - - // Compute additional evacuation supplement, which is extra memory borrowed from old-gen that can be allocated - // by mutators while GC is working on evacuation and update-refs. This memory can be temporarily borrowed - // from old-gen allotment, then repaid at the end of update-refs from the recycled collection set. After - // we have computed the collection set based on the parameters established above, we can make additional - // loans based on our knowledge of the collection set to determine how much allocation we can allow - // during the evacuation and update-refs phases of execution. The total available supplement is the result - // of adding old_bytes_reserved_for_alloc_supplement to the smaller of: - // - // 1. regions_available_to_loan * region_size_bytes - // 2. The replenishment budget (number of regions in collection set - the number of regions already - // under lien for the young_evacuation_reserve) - // - - // Regardless of how many regions may be available to be loaned, we can loan no more regions than - // the total number of young regions to be evacuated. Call this the regions_for_runway. - - if (regions_available_to_loan > 0 && (collected_regions > loaned_regions)) { - assert(regions_available_to_loan * region_size_bytes <= working_old_available, - "regions_available_to_loan should not exceed working_old_available"); - - size_t additional_regions_to_loan = collected_regions - loaned_regions; - if (additional_regions_to_loan > regions_available_to_loan) { - additional_regions_to_loan = regions_available_to_loan; - } - loaned_regions += additional_regions_to_loan; - old_bytes_reserved_for_alloc_supplement += additional_regions_to_loan * region_size_bytes; - working_old_available -= additional_regions_to_loan * region_size_bytes; - } - size_t allocation_supplement = old_bytes_reserved_for_alloc_supplement + old_bytes_loaned_for_young_evac; - assert(allocation_supplement % ShenandoahHeapRegion::region_size_bytes() == 0, - "allocation_supplement must be multiple of region size"); - - heap->set_alloc_supplement_reserve(allocation_supplement); - - // TODO: young_available, which feeds into alloc_budget_evac_and_update is lacking memory available within - // existing young-gen regions that were not selected for the collection set. Add this in and adjust the - // log message (where it says "empty-region allocation budget"). - - - log_debug(gc)("Memory reserved for young evacuation: " SIZE_FORMAT "%s for evacuating " SIZE_FORMAT - "%s out of young available: " SIZE_FORMAT "%s", - byte_size_in_proper_unit(young_evacuated_reserve_used), - proper_unit_for_byte_size(young_evacuated_reserve_used), - byte_size_in_proper_unit(young_evacuated), proper_unit_for_byte_size(young_evacuated), - byte_size_in_proper_unit(young_available), proper_unit_for_byte_size(young_available)); - - log_debug(gc)("Memory reserved for old evacuation: " SIZE_FORMAT "%s for evacuating " SIZE_FORMAT - "%s out of old available: " SIZE_FORMAT "%s", - byte_size_in_proper_unit(old_evacuated), proper_unit_for_byte_size(old_evacuated), - byte_size_in_proper_unit(old_evacuated), proper_unit_for_byte_size(old_evacuated), - byte_size_in_proper_unit(old_available), proper_unit_for_byte_size(old_available)); - - size_t regular_promotion = promotion_reserve - consumed_by_advance_promotion; - size_t excess = - old_available - (old_evacuation_reserve + promotion_reserve + old_bytes_loaned_for_young_evac + allocation_supplement); - log_info(gc, ergo)("Old available: " SIZE_FORMAT "%s is partitioned into old evacuation budget: " SIZE_FORMAT - "%s, aged region promotion budget: " SIZE_FORMAT - "%s, regular region promotion budget: " SIZE_FORMAT - "%s, loaned for young evacuation: " SIZE_FORMAT - "%s, loaned for young allocations: " SIZE_FORMAT - "%s, excess: " SIZE_FORMAT "%s", - byte_size_in_proper_unit(old_available), - proper_unit_for_byte_size(old_available), - byte_size_in_proper_unit(old_evacuation_reserve), - proper_unit_for_byte_size(old_evacuation_reserve), - byte_size_in_proper_unit(consumed_by_advance_promotion), - proper_unit_for_byte_size(consumed_by_advance_promotion), - byte_size_in_proper_unit(regular_promotion), - proper_unit_for_byte_size(regular_promotion), - byte_size_in_proper_unit(old_bytes_loaned_for_young_evac), - proper_unit_for_byte_size(old_bytes_loaned_for_young_evac), - byte_size_in_proper_unit(allocation_supplement), - proper_unit_for_byte_size(allocation_supplement), - byte_size_in_proper_unit(excess), - proper_unit_for_byte_size(excess)); + // Add in the excess_old memory to hold unanticipated promotions, if any. If there are more unanticipated + // promotions than fit in reserved memory, they will be deferred until a future GC pass. + size_t total_promotion_reserve = young_advance_promoted_reserve_used + excess_old; + heap->set_promoted_reserve(total_promotion_reserve); + heap->reset_promoted_expended(); } void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) { @@ -822,7 +487,6 @@ void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) { // Budgeting parameters to compute_evacuation_budgets are passed by reference. compute_evacuation_budgets(heap, preselected_regions, collection_set, consumed_by_advance_promotion); - _heuristics->choose_collection_set(collection_set, heap->old_heuristics()); if (!collection_set->is_empty()) { // only make use of evacuation budgets when we are evacuating @@ -839,7 +503,11 @@ void ShenandoahGeneration::prepare_regions_and_collection_set(bool concurrent) { ShenandoahGCPhase phase(concurrent ? ShenandoahPhaseTimings::final_rebuild_freeset : ShenandoahPhaseTimings::degen_gc_final_rebuild_freeset); ShenandoahHeapLocker locker(heap->lock()); - heap->free_set()->rebuild(); + size_t young_cset_regions, old_cset_regions; + + // We are preparing for evacuation. At this time, we ignore cset region tallies. + heap->free_set()->prepare_to_rebuild(young_cset_regions, old_cset_regions); + heap->free_set()->rebuild(young_cset_regions, old_cset_regions); } heap->set_evacuation_reserve_quantities(false); } @@ -897,7 +565,7 @@ ShenandoahGeneration::ShenandoahGeneration(ShenandoahGenerationType type, _collection_thread_time_s(0.0), _affiliated_region_count(0), _humongous_waste(0), _used(0), _bytes_allocated_since_gc_start(0), _max_capacity(max_capacity), _soft_max_capacity(soft_max_capacity), - _adjusted_capacity(max_capacity), _heuristics(nullptr) { + _heuristics(nullptr) { _is_marking_complete.set(); assert(max_workers > 0, "At least one queue"); for (uint i = 0; i < max_workers; ++i) { @@ -955,6 +623,28 @@ size_t ShenandoahGeneration::decrement_affiliated_region_count() { // on read and write of _affiliated_region_count. At the end of full gc, a single thread overwrites the count with // a coherent value. _affiliated_region_count--; + // TODO: REMOVE IS_GLOBAL() QUALIFIER AFTER WE FIX GLOBAL AFFILIATED REGION ACCOUNTING + assert(is_global() || ShenandoahHeap::heap()->is_full_gc_in_progress() || + (_used + _humongous_waste <= _affiliated_region_count * ShenandoahHeapRegion::region_size_bytes()), + "used + humongous cannot exceed regions"); + return _affiliated_region_count; +} + +size_t ShenandoahGeneration::increase_affiliated_region_count(size_t delta) { + shenandoah_assert_heaplocked_or_fullgc_safepoint(); + _affiliated_region_count += delta; + return _affiliated_region_count; +} + +size_t ShenandoahGeneration::decrease_affiliated_region_count(size_t delta) { + shenandoah_assert_heaplocked_or_fullgc_safepoint(); + assert(_affiliated_region_count > delta, "Affiliated region count cannot be negative"); + + _affiliated_region_count -= delta; + // TODO: REMOVE IS_GLOBAL() QUALIFIER AFTER WE FIX GLOBAL AFFILIATED REGION ACCOUNTING + assert(is_global() || ShenandoahHeap::heap()->is_full_gc_in_progress() || + (_used + _humongous_waste <= _affiliated_region_count * ShenandoahHeapRegion::region_size_bytes()), + "used + humongous cannot exceed regions"); return _affiliated_region_count; } @@ -967,11 +657,11 @@ void ShenandoahGeneration::establish_usage(size_t num_regions, size_t num_bytes, void ShenandoahGeneration::increase_used(size_t bytes) { Atomic::add(&_used, bytes); -} - -void ShenandoahGeneration::decrease_used(size_t bytes) { - assert(_used >= bytes, "cannot reduce bytes used by generation below zero"); - Atomic::sub(&_used, bytes); + // This detects arithmetic wraparound on _used. Non-generational mode does not keep track of _affiliated_region_count + // TODO: REMOVE IS_GLOBAL() QUALIFIER AFTER WE FIX GLOBAL AFFILIATED REGION ACCOUNTING + assert(is_global() || ShenandoahHeap::heap()->is_full_gc_in_progress() || + (_used + _humongous_waste <= _affiliated_region_count * ShenandoahHeapRegion::region_size_bytes()), + "used cannot exceed regions"); } void ShenandoahGeneration::increase_humongous_waste(size_t bytes) { @@ -982,13 +672,25 @@ void ShenandoahGeneration::increase_humongous_waste(size_t bytes) { void ShenandoahGeneration::decrease_humongous_waste(size_t bytes) { if (bytes > 0) { - assert(_humongous_waste >= bytes, "Waste cannot be negative"); assert(ShenandoahHeap::heap()->is_full_gc_in_progress() || (_humongous_waste >= bytes), "Waste (" SIZE_FORMAT ") cannot be negative (after subtracting " SIZE_FORMAT ")", _humongous_waste, bytes); Atomic::sub(&_humongous_waste, bytes); } } +void ShenandoahGeneration::decrease_used(size_t bytes) { + // TODO: REMOVE IS_GLOBAL() QUALIFIER AFTER WE FIX GLOBAL AFFILIATED REGION ACCOUNTING + assert(is_global() || ShenandoahHeap::heap()->is_full_gc_in_progress() || + (_used >= bytes), "cannot reduce bytes used by generation below zero"); + Atomic::sub(&_used, bytes); + + // Non-generational mode does not maintain affiliated region counts + // TODO: REMOVE IS_GLOBAL() QUALIFIER AFTER WE FIX GLOBAL AFFILIATED REGION ACCOUNTING + assert(is_global() || ShenandoahHeap::heap()->is_full_gc_in_progress() || + (_affiliated_region_count * ShenandoahHeapRegion::region_size_bytes() >= _used), + "Affiliated regions must hold more than what is currently used"); +} + size_t ShenandoahGeneration::used_regions() const { return _affiliated_region_count; } @@ -996,7 +698,7 @@ size_t ShenandoahGeneration::used_regions() const { size_t ShenandoahGeneration::free_unaffiliated_regions() const { size_t result = max_capacity() / ShenandoahHeapRegion::region_size_bytes(); if (_affiliated_region_count > result) { - result = 0; // If old-gen is loaning regions to young-gen, affiliated regions may exceed capacity temporarily. + result = 0; } else { result -= _affiliated_region_count; } @@ -1019,61 +721,48 @@ size_t ShenandoahGeneration::soft_available() const { return in_use > soft_capacity ? 0 : soft_capacity - in_use; } -size_t ShenandoahGeneration::adjust_available(intptr_t adjustment) { - assert(adjustment % ShenandoahHeapRegion::region_size_bytes() == 0, - "Adjustment to generation size must be multiple of region size"); - _adjusted_capacity = max_capacity() + adjustment; - return _adjusted_capacity; -} - -size_t ShenandoahGeneration::unadjust_available() { - _adjusted_capacity = max_capacity(); - return _adjusted_capacity; -} - -size_t ShenandoahGeneration::adjusted_available() const { - size_t in_use = used() + get_humongous_waste(); - size_t capacity = _adjusted_capacity; - return in_use > capacity ? 0 : capacity - in_use; -} - -size_t ShenandoahGeneration::adjusted_capacity() const { - return _adjusted_capacity; -} - -size_t ShenandoahGeneration::adjusted_unaffiliated_regions() const { - // This assertion has been disabled because we expect this code to be replaced by 05/2023 - // assert(adjusted_capacity() >= used_regions_size(), "adjusted_unaffiliated_regions() cannot return negative"); - assert((adjusted_capacity() - used_regions_size()) % ShenandoahHeapRegion::region_size_bytes() == 0, - "adjusted capacity (" SIZE_FORMAT ") and used regions size (" SIZE_FORMAT ") should be multiples of region_size_bytes", - adjusted_capacity(), used_regions_size()); - return (adjusted_capacity() - used_regions_size()) / ShenandoahHeapRegion::region_size_bytes(); -} - void ShenandoahGeneration::increase_capacity(size_t increment) { shenandoah_assert_heaplocked_or_safepoint(); - assert(_max_capacity + increment <= ShenandoahHeap::heap()->max_size_for(this), "Cannot increase generation capacity beyond maximum."); - assert(increment % ShenandoahHeapRegion::region_size_bytes() == 0, "Region-sized changes only"); - // TODO: ysr: remove this check and warning - if (increment % ShenandoahHeapRegion::region_size_bytes() != 0) { - log_warning(gc)("Increment (" INTPTR_FORMAT ") should be a multiple of region size (" SIZE_FORMAT ")", - increment, ShenandoahHeapRegion::region_size_bytes()); - } + + // We do not enforce that new capacity >= heap->max_size_for(this). The maximum generation size is treated as a rule of thumb + // which may be violated during certain transitions, such as when we are forcing transfers for the purpose of promoting regions + // in place. + // TODO: REMOVE IS_GLOBAL() QUALIFIER AFTER WE FIX GLOBAL AFFILIATED REGION ACCOUNTING + assert(is_global() || ShenandoahHeap::heap()->is_full_gc_in_progress() || + (_max_capacity + increment <= ShenandoahHeap::heap()->max_capacity()), "Generation cannot be larger than heap size"); + assert(increment % ShenandoahHeapRegion::region_size_bytes() == 0, "Generation capacity must be multiple of region size"); _max_capacity += increment; - _adjusted_capacity += increment; + + // This detects arithmetic wraparound on _used + // TODO: REMOVE IS_GLOBAL() QUALIFIER AFTER WE FIX GLOBAL AFFILIATED REGION ACCOUNTING + assert(is_global() || ShenandoahHeap::heap()->is_full_gc_in_progress() || + (_affiliated_region_count * ShenandoahHeapRegion::region_size_bytes() >= _used), + "Affiliated regions must hold more than what is currently used"); } void ShenandoahGeneration::decrease_capacity(size_t decrement) { shenandoah_assert_heaplocked_or_safepoint(); - assert(_max_capacity - decrement >= ShenandoahHeap::heap()->min_size_for(this), "Cannot decrease generation capacity beyond minimum."); - assert(decrement % ShenandoahHeapRegion::region_size_bytes() == 0, "Region-sized changes only"); - // TODO: ysr: remove this check and warning - if (decrement % ShenandoahHeapRegion::region_size_bytes() != 0) { - log_warning(gc)("Decrement (" INTPTR_FORMAT ") should be a multiple of region size (" SIZE_FORMAT ")", - decrement, ShenandoahHeapRegion::region_size_bytes()); - } + + // We do not enforce that new capacity >= heap->min_size_for(this). The minimum generation size is treated as a rule of thumb + // which may be violated during certain transitions, such as when we are forcing transfers for the purpose of promoting regions + // in place. + assert(decrement % ShenandoahHeapRegion::region_size_bytes() == 0, "Generation capacity must be multiple of region size"); + assert(_max_capacity >= decrement, "Generation capacity cannot be negative"); + _max_capacity -= decrement; - _adjusted_capacity -= decrement; + + // This detects arithmetic wraparound on _used + // TODO: REMOVE IS_GLOBAL() QUALIFIER AFTER WE FIX GLOBAL AFFILIATED REGION ACCOUNTING + assert(is_global() || ShenandoahHeap::heap()->is_full_gc_in_progress() || + (_affiliated_region_count * ShenandoahHeapRegion::region_size_bytes() >= _used), + "Affiliated regions must hold more than what is currently used"); + // TODO: REMOVE IS_GLOBAL() QUALIFIER AFTER WE FIX GLOBAL AFFILIATED REGION ACCOUNTING + assert(is_global() || ShenandoahHeap::heap()->is_full_gc_in_progress() || + (_used <= _max_capacity), "Cannot use more than capacity"); + // TODO: REMOVE IS_GLOBAL() QUALIFIER AFTER WE FIX GLOBAL AFFILIATED REGION ACCOUNTING + assert(is_global() || ShenandoahHeap::heap()->is_full_gc_in_progress() || + (_affiliated_region_count * ShenandoahHeapRegion::region_size_bytes() <= _max_capacity), + "Cannot use more than capacity"); } void ShenandoahGeneration::record_success_concurrent(bool abbreviated) { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp index 160a51a443b98..2bd854142802c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.hpp @@ -50,8 +50,6 @@ class ShenandoahGeneration : public CHeapObj { double _collection_thread_time_s; -protected: - // Usage size_t _affiliated_region_count; // How much free memory is left in the last region of humongous objects. @@ -61,13 +59,15 @@ class ShenandoahGeneration : public CHeapObj { // The units are bytes. The value is only changed on a safepoint or under the // heap lock. size_t _humongous_waste; + +protected: + // Usage + volatile size_t _used; volatile size_t _bytes_allocated_since_gc_start; size_t _max_capacity; size_t _soft_max_capacity; - size_t _adjusted_capacity; - ShenandoahHeuristics* _heuristics; private: @@ -117,22 +117,6 @@ class ShenandoahGeneration : public CHeapObj { // max heap size will cause the adaptive heuristic to run more frequent cycles. size_t soft_available() const; - // During evacuation and update-refs, some memory may be shifted between generations. In particular, memory - // may be loaned by old-gen to young-gen based on the promise the loan will be promptly repaid from the memory reclaimed - // when the current collection set is recycled. The capacity adjustment also takes into consideration memory that is - // set aside within each generation to hold the results of evacuation, but not promotion, into that region. Promotions - // into old-gen are bounded by adjusted_available() whereas evacuations into old-gen are pre-committed. - size_t adjusted_available() const; - size_t adjusted_capacity() const; - - // This is the number of FREE regions that are eligible to be affiliated with this generation according to the current - // adjusted capacity. - size_t adjusted_unaffiliated_regions() const; - - // Both of following return new value of available - size_t adjust_available(intptr_t adjustment); - size_t unadjust_available(); - size_t bytes_allocated_since_gc_start(); void reset_bytes_allocated_since_gc_start(); void increase_allocated(size_t bytes); @@ -205,6 +189,12 @@ class ShenandoahGeneration : public CHeapObj { // Return the updated value of affiliated_region_count size_t decrement_affiliated_region_count(); + // Return the updated value of affiliated_region_count + size_t increase_affiliated_region_count(size_t delta); + + // Return the updated value of affiliated_region_count + size_t decrease_affiliated_region_count(size_t delta); + void establish_usage(size_t num_regions, size_t num_bytes, size_t humongous_waste); void increase_used(size_t bytes); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 348e1900e3984..eaa510099555e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -379,8 +379,11 @@ jint ShenandoahHeap::initialize() { // Initialize to complete _marking_context->mark_complete(); + size_t young_cset_regions, old_cset_regions; - _free_set->rebuild(); + // We are initializing free set. We ignore cset region tallies. + _free_set->prepare_to_rebuild(young_cset_regions, old_cset_regions); + _free_set->rebuild(young_cset_regions, old_cset_regions); } if (AlwaysPreTouch) { @@ -519,15 +522,10 @@ void ShenandoahHeap::initialize_heuristics_generations() { _young_generation = new ShenandoahYoungGeneration(_max_workers, max_capacity_young, initial_capacity_young); _old_generation = new ShenandoahOldGeneration(_max_workers, max_capacity_old, initial_capacity_old); - _global_generation = new ShenandoahGlobalGeneration(_gc_mode->is_generational(), _max_workers, soft_max_capacity(), soft_max_capacity()); - + _global_generation = new ShenandoahGlobalGeneration(_gc_mode->is_generational(), _max_workers, max_capacity(), max_capacity()); _global_generation->initialize_heuristics(_gc_mode); - if (mode()->is_generational()) { - _young_generation->initialize_heuristics(_gc_mode); - _old_generation->initialize_heuristics(_gc_mode); - - ShenandoahEvacWaste = ShenandoahGenerationalEvacWaste; - } + _young_generation->initialize_heuristics(_gc_mode); + _old_generation->initialize_heuristics(_gc_mode); } #ifdef _MSC_VER @@ -540,6 +538,8 @@ ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) : _gc_generation(nullptr), _prepare_for_old_mark(false), _initial_size(0), + _promotion_potential(0), + _promotion_in_place_potential(0), _committed(0), _max_workers(MAX3(ConcGCThreads, ParallelGCThreads, 1U)), _workers(nullptr), @@ -549,7 +549,6 @@ ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) : _regions(nullptr), _affiliations(nullptr), _update_refs_iterator(this), - _alloc_supplement_reserve(0), _promoted_reserve(0), _old_evac_reserve(0), _old_evac_expended(0), @@ -581,6 +580,8 @@ ShenandoahHeap::ShenandoahHeap(ShenandoahCollectorPolicy* policy) : _gc_timer(new ConcurrentGCTimer()), _soft_ref_policy(), _log_min_obj_alignment_in_bytes(LogMinObjAlignmentInBytes), + _old_regions_surplus(0), + _old_regions_deficit(0), _marking_context(nullptr), _bitmap_size(0), _bitmap_regions_per_slice(0), @@ -818,13 +819,6 @@ void ShenandoahHeap::set_soft_max_capacity(size_t v) { "Should be in bounds: " SIZE_FORMAT " <= " SIZE_FORMAT " <= " SIZE_FORMAT, min_capacity(), v, max_capacity()); Atomic::store(&_soft_max_size, v); - - if (mode()->is_generational()) { - size_t max_capacity_young = _generation_sizer.max_young_size(); - size_t min_capacity_young = _generation_sizer.min_young_size(); - size_t new_capacity_young = clamp(v, min_capacity_young, max_capacity_young); - _young_generation->set_soft_max_capacity(new_capacity_young); - } } size_t ShenandoahHeap::min_capacity() const { @@ -893,9 +887,7 @@ void ShenandoahHeap::handle_promotion_failure() { } void ShenandoahHeap::report_promotion_failure(Thread* thread, size_t size) { - // We squelch excessive reports to reduce noise in logs. Squelch enforcement is not "perfect" because - // this same code can be in-lined in multiple contexts, and each context will have its own copy of the static - // last_report_epoch and this_epoch_report_count variables. + // We squelch excessive reports to reduce noise in logs. const size_t MaxReportsPerEpoch = 4; static size_t last_report_epoch = 0; static size_t epoch_report_count = 0; @@ -915,11 +907,18 @@ void ShenandoahHeap::report_promotion_failure(Thread* thread, size_t size) { PLAB* plab = ShenandoahThreadLocalData::plab(thread); size_t words_remaining = (plab == nullptr)? 0: plab->words_remaining(); const char* promote_enabled = ShenandoahThreadLocalData::allow_plab_promotions(thread)? "enabled": "disabled"; + ShenandoahGeneration* old_gen = old_generation(); + size_t old_capacity = old_gen->max_capacity(); + size_t old_usage = old_gen->used(); + size_t old_free_regions = old_gen->free_unaffiliated_regions(); log_info(gc, ergo)("Promotion failed, size " SIZE_FORMAT ", has plab? %s, PLAB remaining: " SIZE_FORMAT - ", plab promotions %s, promotion reserve: " SIZE_FORMAT ", promotion expended: " SIZE_FORMAT, - size, plab == nullptr? "no": "yes", - words_remaining, promote_enabled, promotion_reserve, promotion_expended); + ", plab promotions %s, promotion reserve: " SIZE_FORMAT ", promotion expended: " SIZE_FORMAT + ", old capacity: " SIZE_FORMAT ", old_used: " SIZE_FORMAT ", old unaffiliated regions: " SIZE_FORMAT, + size * HeapWordSize, plab == nullptr? "no": "yes", + words_remaining * HeapWordSize, promote_enabled, promotion_reserve, promotion_expended, + old_capacity, old_usage, old_free_regions); + if ((gc_id == last_report_epoch) && (epoch_report_count >= MaxReportsPerEpoch)) { log_info(gc, ergo)("Squelching additional promotion failure reports for current epoch"); } else if (gc_id != last_report_epoch) { @@ -1029,7 +1028,6 @@ HeapWord* ShenandoahHeap::allocate_from_plab_slow(Thread* thread, size_t size, b // CAUTION: retire_plab may register the remnant filler object with the remembered set scanner without a lock. This // is safe iff it is assured that each PLAB is a whole-number multiple of card-mark memory size and each PLAB is // aligned with the start of a card's memory range. - retire_plab(plab, thread); size_t actual_size = 0; @@ -1037,7 +1035,12 @@ HeapWord* ShenandoahHeap::allocate_from_plab_slow(Thread* thread, size_t size, b // less than the remaining evacuation need. It also adjusts plab_preallocated and expend_promoted if appropriate. HeapWord* plab_buf = allocate_new_plab(min_size, cur_size, &actual_size); if (plab_buf == nullptr) { - return nullptr; + if (min_size == PLAB::min_size()) { + // Disable plab promotions for this thread because we cannot even allocate a plab of minimal size. This allows us + // to fail faster on subsequent promotion attempts. + ShenandoahThreadLocalData::disable_plab_promotions(thread); + } + return NULL; } else { ShenandoahThreadLocalData::enable_plab_retries(thread); } @@ -1056,7 +1059,6 @@ HeapWord* ShenandoahHeap::allocate_from_plab_slow(Thread* thread, size_t size, b #endif // ASSERT } plab->set_buf(plab_buf, actual_size); - if (is_promotion && !ShenandoahThreadLocalData::allow_plab_promotions(thread)) { return nullptr; } @@ -1154,11 +1156,93 @@ void ShenandoahHeap::coalesce_and_fill_old_regions() { parallel_heap_region_iterate(&coalesce); } -bool ShenandoahHeap::adjust_generation_sizes() { - if (mode()->is_generational()) { - return _generation_sizer.adjust_generation_sizes(); +// xfer_limit is the maximum we're able to transfer from young to old +void ShenandoahHeap::adjust_generation_sizes_for_next_cycle( + size_t xfer_limit, size_t young_cset_regions, size_t old_cset_regions) { + + // Make sure old-generation is large enough, but no larger, than is necessary to hold mixed evacuations + // and promotions if we anticipate either. + size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); + size_t promo_load = get_promotion_potential(); + // The free set will reserve this amount of memory to hold young evacuations + size_t young_reserve = (young_generation()->max_capacity() * ShenandoahEvacReserve) / 100; + size_t old_reserve = 0; + size_t mixed_candidates = old_heuristics()->unprocessed_old_collection_candidates(); + bool doing_mixed = (mixed_candidates > 0); + bool doing_promotions = promo_load > 0; + + // round down + size_t max_old_region_xfer = xfer_limit / region_size_bytes; + + // We can limit the reserve to the size of anticipated promotions + size_t max_old_reserve = young_reserve * ShenandoahOldEvacRatioPercent / (100 - ShenandoahOldEvacRatioPercent); + // Here's the algebra: + // TotalEvacuation = OldEvacuation + YoungEvacuation + // OldEvacuation = TotalEvacuation*(ShenandoahOldEvacRatioPercent/100) + // OldEvacuation = YoungEvacuation * (ShenandoahOldEvacRatioPercent/100)/(1 - ShenandoahOldEvacRatioPercent/100) + // OldEvacuation = YoungEvacuation * ShenandoahOldEvacRatioPercent/(100 - ShenandoahOldEvacRatioPercent) + + size_t reserve_for_mixed, reserve_for_promo; + if (doing_mixed) { + assert(old_generation()->available() >= old_generation()->free_unaffiliated_regions() * region_size_bytes, + "Unaffiliated available must be less than total available"); + + // We want this much memory to be unfragmented in order to reliably evacuate old. This is conservative because we + // may not evacuate the entirety of unprocessed candidates in a single mixed evacuation. + size_t max_evac_need = (size_t) + (old_heuristics()->unprocessed_old_collection_candidates_live_memory() * ShenandoahOldEvacWaste); + size_t old_fragmented_available = + old_generation()->available() - old_generation()->free_unaffiliated_regions() * region_size_bytes; + reserve_for_mixed = max_evac_need + old_fragmented_available; + if (reserve_for_mixed > max_old_reserve) { + reserve_for_mixed = max_old_reserve; + } + } else { + reserve_for_mixed = 0; } - return false; + + size_t available_for_promotions = max_old_reserve - reserve_for_mixed; + if (doing_promotions) { + // We're only promoting and we have a maximum bound on the amount to be promoted + reserve_for_promo = (size_t) (promo_load * ShenandoahPromoEvacWaste); + if (reserve_for_promo > available_for_promotions) { + reserve_for_promo = available_for_promotions; + } + } else { + reserve_for_promo = 0; + } + old_reserve = reserve_for_mixed + reserve_for_promo; + assert(old_reserve <= max_old_reserve, "cannot reserve more than max for old evacuations"); + size_t old_available = old_generation()->available() + old_cset_regions * region_size_bytes; + size_t young_available = young_generation()->available() + young_cset_regions * region_size_bytes; + size_t old_region_deficit = 0; + size_t old_region_surplus = 0; + if (old_available >= old_reserve) { + size_t old_excess = old_available - old_reserve; + size_t excess_regions = old_excess / region_size_bytes; + size_t unaffiliated_old_regions = old_generation()->free_unaffiliated_regions() + old_cset_regions; + size_t unaffiliated_old = unaffiliated_old_regions * region_size_bytes; + if (unaffiliated_old_regions < excess_regions) { + // We'll give only unaffiliated old to young, which is known to be less than the excess. + old_region_surplus = unaffiliated_old_regions; + } else { + // unaffiliated_old_regions > excess_regions, so we only give away the excess. + old_region_surplus = excess_regions; + } + } else { + // We need to request transfer from YOUNG. Ignore that this will directly impact young_generation()->max_capacity(), + // indirectly impacting young_reserve and old_reserve. These computations are conservative. + size_t old_need = old_reserve - old_available; + // Round up the number of regions needed from YOUNG + old_region_deficit = (old_need + region_size_bytes - 1) / region_size_bytes; + } + if (old_region_deficit > max_old_region_xfer) { + // If we're running short on young-gen memory, limit the xfer. Old-gen collection activities will be curtailed + // if the budget is smaller than desired. + old_region_deficit = max_old_region_xfer; + } + set_old_region_surplus(old_region_surplus); + set_old_region_deficit(old_region_deficit); } // Called from stubs in JIT code or interpreter @@ -1193,7 +1277,7 @@ HeapWord* ShenandoahHeap::allocate_new_plab(size_t min_size, size_t* actual_size) { ShenandoahAllocRequest req = ShenandoahAllocRequest::for_plab(min_size, word_size); // Note that allocate_memory() sets a thread-local flag to prohibit further promotions by this thread - // if we are at risk of exceeding the old-gen evacuation budget. + // if we are at risk of infringing on the old-gen evacuation budget. HeapWord* res = allocate_memory(req, false); if (res != nullptr) { *actual_size = req.actual_size(); @@ -1228,14 +1312,16 @@ HeapWord* ShenandoahHeap::allocate_memory(ShenandoahAllocRequest& req, bool is_p // strategy is to try again, as long as GC makes progress. // // Then, we need to make sure the allocation was retried after at least one - // Full GC, which means we want to try more than ShenandoahFullGCThreshold times. + // Full GC. size_t tries = 0; + size_t original_fullgc_count = shenandoah_policy()->get_fullgc_count(); while (result == nullptr && _progress_last_gc.is_set()) { tries++; control_thread()->handle_alloc_failure(req); result = allocate_memory_under_lock(req, in_new_region, is_promotion); } - while (result == nullptr && tries <= ShenandoahFullGCThreshold) { + while (result == nullptr && + ((shenandoah_policy()->get_fullgc_count() == original_fullgc_count) || (tries <= ShenandoahOOMGCRetries))) { tries++; control_thread()->handle_alloc_failure(req); result = allocate_memory_under_lock(req, in_new_region, is_promotion); @@ -1297,20 +1383,18 @@ HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req if (mode()->is_generational()) { if (req.affiliation() == YOUNG_GENERATION) { if (req.is_mutator_alloc()) { - size_t young_available = young_generation()->adjusted_available(); - if (requested_bytes > young_available) { - // We know this is not a GCLAB. This must be a TLAB or a shared allocation. - if (req.is_lab_alloc() && (young_available >= req.min_size())) { - try_smaller_lab_size = true; - smaller_lab_size = young_available / HeapWordSize; - } else { - // Can't allocate because even min_size() is larger than remaining young_available - log_info(gc, ergo)("Unable to shrink %s alloc request of minimum size: " SIZE_FORMAT - ", young available: " SIZE_FORMAT, - req.is_lab_alloc()? "TLAB": "shared", - HeapWordSize * (req.is_lab_alloc()? req.min_size(): req.size()), young_available); - return nullptr; - } + size_t young_words_available = young_generation()->available() / HeapWordSize; + if (ShenandoahElasticTLAB && req.is_lab_alloc() && (req.min_size() < young_words_available)) { + // Allow ourselves to try a smaller lab size even if requested_bytes <= young_available. We may need a smaller + // lab size because young memory has become too fragmented. + try_smaller_lab_size = true; + smaller_lab_size = (young_words_available < req.size())? young_words_available: req.size(); + } else if (req.size() > young_words_available) { + // Can't allocate because even min_size() is larger than remaining young_available + log_info(gc, ergo)("Unable to shrink %s alloc request of minimum size: " SIZE_FORMAT + ", young words available: " SIZE_FORMAT, req.type_string(), + HeapWordSize * (req.is_lab_alloc()? req.min_size(): req.size()), young_words_available); + return nullptr; } } } else { // reg.affiliation() == OLD_GENERATION @@ -1351,66 +1435,83 @@ HeapWord* ShenandoahHeap::allocate_memory_under_lock(ShenandoahAllocRequest& req } } // This ends the is_generational() block - if (!try_smaller_lab_size) { - result = (allow_allocation)? _free_set->allocate(req, in_new_region): nullptr; - if (result != nullptr) { - if (req.is_old()) { - ShenandoahThreadLocalData::reset_plab_promoted(thread); - if (req.is_gc_alloc()) { - if (req.type() == ShenandoahAllocRequest::_alloc_plab) { - if (promotion_eligible) { - size_t actual_size = req.actual_size() * HeapWordSize; + // First try the original request. If TLAB request size is greater than available, allocate() will attempt to downsize + // request to fit within available memory. + result = (allow_allocation)? _free_set->allocate(req, in_new_region): nullptr; + if (result != nullptr) { + if (req.is_old()) { + ShenandoahThreadLocalData::reset_plab_promoted(thread); + if (req.is_gc_alloc()) { + bool disable_plab_promotions = false; + if (req.type() == ShenandoahAllocRequest::_alloc_plab) { + if (promotion_eligible) { + size_t actual_size = req.actual_size() * HeapWordSize; + // The actual size of the allocation may be larger than the requested bytes (due to alignment on card boundaries). + // If this puts us over our promotion budget, we need to disable future PLAB promotions for this thread. + if (get_promoted_expended() + actual_size <= get_promoted_reserve()) { // Assume the entirety of this PLAB will be used for promotion. This prevents promotion from overreach. // When we retire this plab, we'll unexpend what we don't really use. ShenandoahThreadLocalData::enable_plab_promotions(thread); expend_promoted(actual_size); - // This assert has been disabled because we expect this code to be replaced by 05/2023. - // assert(get_promoted_expended() <= get_promoted_reserve(), "Do not expend more promotion than budgeted"); + assert(get_promoted_expended() <= get_promoted_reserve(), "Do not expend more promotion than budgeted"); ShenandoahThreadLocalData::set_plab_preallocated_promoted(thread, actual_size); } else { - // Disable promotions in this thread because entirety of this PLAB must be available to hold old-gen evacuations. - ShenandoahThreadLocalData::disable_plab_promotions(thread); - ShenandoahThreadLocalData::set_plab_preallocated_promoted(thread, 0); + disable_plab_promotions = true; } - } else if (is_promotion) { - // Shared promotion. Assume size is requested_bytes. - expend_promoted(requested_bytes); - assert(get_promoted_expended() <= get_promoted_reserve(), "Do not expend more promotion than budgeted"); + } else { + disable_plab_promotions = true; } + if (disable_plab_promotions) { + // Disable promotions in this thread because entirety of this PLAB must be available to hold old-gen evacuations. + ShenandoahThreadLocalData::disable_plab_promotions(thread); + ShenandoahThreadLocalData::set_plab_preallocated_promoted(thread, 0); + } + } else if (is_promotion) { + // Shared promotion. Assume size is requested_bytes. + expend_promoted(requested_bytes); + assert(get_promoted_expended() <= get_promoted_reserve(), "Do not expend more promotion than budgeted"); } - - // Register the newly allocated object while we're holding the global lock since there's no synchronization - // built in to the implementation of register_object(). There are potential races when multiple independent - // threads are allocating objects, some of which might span the same card region. For example, consider - // a card table's memory region within which three objects are being allocated by three different threads: - // - // objects being "concurrently" allocated: - // [-----a------][-----b-----][--------------c------------------] - // [---- card table memory range --------------] - // - // Before any objects are allocated, this card's memory range holds no objects. Note that allocation of object a - // wants to set the has-object, first-start, and last-start attributes of the preceding card region. - // allocation of object b wants to set the has-object, first-start, and last-start attributes of this card region. - // allocation of object c also wants to set the has-object, first-start, and last-start attributes of this card region. - // - // The thread allocating b and the thread allocating c can "race" in various ways, resulting in confusion, such as - // last-start representing object b while first-start represents object c. This is why we need to require all - // register_object() invocations to be "mutually exclusive" with respect to each card's memory range. - ShenandoahHeap::heap()->card_scan()->register_object(result); - } - } else { - // The allocation failed. If this was a plab allocation, We've already retired it and no longer have a plab. - if (req.is_old() && req.is_gc_alloc() && - (req.type() == ShenandoahAllocRequest::_alloc_plab)) { - // We don't need to disable PLAB promotions because there is no PLAB. We leave promotions enabled because - // this allows the surrounding infrastructure to retry alloc_plab_slow() with a smaller PLAB size. - ShenandoahThreadLocalData::set_plab_preallocated_promoted(thread, 0); } + + // Register the newly allocated object while we're holding the global lock since there's no synchronization + // built in to the implementation of register_object(). There are potential races when multiple independent + // threads are allocating objects, some of which might span the same card region. For example, consider + // a card table's memory region within which three objects are being allocated by three different threads: + // + // objects being "concurrently" allocated: + // [-----a------][-----b-----][--------------c------------------] + // [---- card table memory range --------------] + // + // Before any objects are allocated, this card's memory range holds no objects. Note that allocation of object a + // wants to set the starts-object, first-start, and last-start attributes of the preceding card region. + // allocation of object b wants to set the starts-object, first-start, and last-start attributes of this card region. + // allocation of object c also wants to set the starts-object, first-start, and last-start attributes of this + // card region. + // + // The thread allocating b and the thread allocating c can "race" in various ways, resulting in confusion, such as + // last-start representing object b while first-start represents object c. This is why we need to require all + // register_object() invocations to be "mutually exclusive" with respect to each card's memory range. + ShenandoahHeap::heap()->card_scan()->register_object(result); + } + } else { + // The allocation failed. If this was a plab allocation, We've already retired it and no longer have a plab. + if (req.is_old() && req.is_gc_alloc() && (req.type() == ShenandoahAllocRequest::_alloc_plab)) { + // We don't need to disable PLAB promotions because there is no PLAB. We leave promotions enabled because + // this allows the surrounding infrastructure to retry alloc_plab_slow() with a smaller PLAB size. + ShenandoahThreadLocalData::set_plab_preallocated_promoted(thread, 0); } + } + if ((result != nullptr) || !try_smaller_lab_size) { return result; } - // else, try_smaller_lab_size is true so we fall through and recurse with a smaller lab size - } // This closes the block that holds the heap lock. This releases the lock. + // else, fall through to try_smaller_lab_size + } // This closes the block that holds the heap lock, releasing the lock. + + // We failed to allocate the originally requested lab size. Let's see if we can allocate a smaller lab size. + if (req.size() == smaller_lab_size) { + // If we were already trying to allocate min size, no value in attempting to repeat the same. End the recursion. + return nullptr; + } // We arrive here if the tlab allocation request can be resized to fit within young_available assert((req.affiliation() == YOUNG_GENERATION) && req.is_lab_alloc() && req.is_mutator_alloc() && @@ -1589,31 +1690,41 @@ class ShenandoahGenerationalEvacuationTask : public WorkerTask { void do_work() { ShenandoahConcurrentEvacuateRegionObjectClosure cl(_sh); ShenandoahHeapRegion* r; + ShenandoahMarkingContext* const ctx = ShenandoahHeap::heap()->marking_context(); + size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); + size_t old_garbage_threshold = (region_size_bytes * ShenandoahOldGarbageThreshold) / 100; while ((r = _regions->next()) != nullptr) { - log_debug(gc)("GenerationalEvacuationTask do_work(), looking at %s region " SIZE_FORMAT ", (age: %d) [%s, %s]", + log_debug(gc)("GenerationalEvacuationTask do_work(), looking at %s region " SIZE_FORMAT ", (age: %d) [%s, %s, %s]", r->is_old()? "old": r->is_young()? "young": "free", r->index(), r->age(), r->is_active()? "active": "inactive", - r->is_humongous()? (r->is_humongous_start()? "humongous_start": "humongous_continuation"): "regular"); + r->is_humongous()? (r->is_humongous_start()? "humongous_start": "humongous_continuation"): "regular", + r->is_cset()? "cset": "not-cset"); + if (r->is_cset()) { assert(r->has_live(), "Region " SIZE_FORMAT " should have been reclaimed early", r->index()); _sh->marked_object_iterate(r, &cl); if (ShenandoahPacing) { _sh->pacer()->report_evac(r->used() >> LogHeapWordSize); } - } else if (r->is_young() && r->is_active() && r->is_humongous_start() && (r->age() > InitialTenuringThreshold)) { - // We promote humongous_start regions along with their affiliated continuations during evacuation rather than - // doing this work during a safepoint. We cannot put humongous regions into the collection set because that - // triggers the load-reference barrier (LRB) to copy on reference fetch. - if (r->promote_humongous() == 0) { - // We chose not to promote because old-gen is out of memory. Report and handle the promotion failure because - // this suggests need for expanding old-gen and/or performing collection of old-gen. - ShenandoahHeap* heap = ShenandoahHeap::heap(); - oop obj = cast_to_oop(r->bottom()); - size_t size = obj->size(); - Thread* thread = Thread::current(); - heap->report_promotion_failure(thread, size); - heap->handle_promotion_failure(); + } else if (r->is_young() && r->is_active() && (r->age() >= InitialTenuringThreshold)) { + HeapWord* tams = ctx->top_at_mark_start(r); + if (r->is_humongous_start()) { + // We promote humongous_start regions along with their affiliated continuations during evacuation rather than + // doing this work during a safepoint. We cannot put humongous regions into the collection set because that + // triggers the load-reference barrier (LRB) to copy on reference fetch. + r->promote_humongous(); + } else if (r->is_regular() && (r->garbage_before_padded_for_promote() < old_garbage_threshold) && (r->get_top_before_promote() == tams)) { + // Likewise, we cannot put promote-in-place regions into the collection set because that would also trigger + // the LRB to copy on reference fetch. + r->promote_in_place(); } + // Aged humongous continuation regions are handled with their start region. If an aged regular region has + // more garbage than ShenandoahOldGarbageTrheshold, we'll promote by evacuation. If there is room for evacuation + // in this cycle, the region will be in the collection set. If there is not room, the region will be promoted + // by evacuation in some future GC cycle. + + // If an aged regular region has received allocations during the current cycle, we do not promote because the + // newly allocated objects do not have appropriate age; this region's age will be reset to zero at end of cycle. } // else, region is free, or OLD, or not in collection set, or humongous_continuation, // or is young humongous_start that is too young to be promoted @@ -1812,7 +1923,7 @@ void ShenandoahHeap::set_young_lab_region_flags() { size_t ShenandoahHeap::unsafe_max_tlab_alloc(Thread *thread) const { if (ShenandoahElasticTLAB) { if (mode()->is_generational()) { - return MIN2(ShenandoahHeapRegion::max_tlab_size_bytes(), young_generation()->adjusted_available()); + return MIN2(ShenandoahHeapRegion::max_tlab_size_bytes(), young_generation()->available()); } else { // With Elastic TLABs, return the max allowed size, and let the allocation path // figure out the safe size for current allocation. @@ -1900,24 +2011,16 @@ void ShenandoahHeap::on_cycle_start(GCCause::Cause cause, ShenandoahGeneration* shenandoah_policy()->record_cycle_start(); generation->heuristics()->record_cycle_start(); - - // When a cycle starts, attribute any thread activity when the collector - // is idle to the global generation. - _mmu_tracker.record(global_generation()); } void ShenandoahHeap::on_cycle_end(ShenandoahGeneration* generation) { generation->heuristics()->record_cycle_end(); - if (mode()->is_generational() && (generation->is_global() || upgraded_to_full())) { // If we just completed a GLOBAL GC, claim credit for completion of young-gen and old-gen GC as well young_generation()->heuristics()->record_cycle_end(); old_generation()->heuristics()->record_cycle_end(); } set_gc_cause(GCCause::_no_gc); - - // When a cycle ends, the thread activity is attributed to the respective generation - _mmu_tracker.record(generation); } void ShenandoahHeap::verify(VerifyOption vo) { @@ -2676,6 +2779,16 @@ class ShenandoahUpdateHeapRefsTask : public WorkerTask { template void do_work(uint worker_id) { T cl; + if (CONCURRENT && (worker_id == 0)) { + // We ask the first worker to replenish the Mutator free set by moving regions previously reserved to hold the + // results of evacuation. These reserves are no longer necessary because evacuation has completed. + size_t cset_regions = _heap->collection_set()->count(); + // We cannot transfer any more regions than will be reclaimed when the existing collection set is recycled, because + // we need the reclaimed collection set regions to replenish the collector reserves + _heap->free_set()->move_collector_sets_to_mutator(cset_regions); + } + // If !CONCURRENT, there's no value in expanding Mutator free set + ShenandoahHeapRegion* r = _regions->next(); // We update references for global, old, and young collections. assert(_heap->active_generation()->is_mark_complete(), "Expected complete marking"); @@ -2938,12 +3051,58 @@ void ShenandoahHeap::update_heap_region_states(bool concurrent) { } void ShenandoahHeap::rebuild_free_set(bool concurrent) { - { - ShenandoahGCPhase phase(concurrent ? - ShenandoahPhaseTimings::final_update_refs_rebuild_freeset : - ShenandoahPhaseTimings::degen_gc_final_update_refs_rebuild_freeset); - ShenandoahHeapLocker locker(lock()); - _free_set->rebuild(); + ShenandoahGCPhase phase(concurrent ? + ShenandoahPhaseTimings::final_update_refs_rebuild_freeset : + ShenandoahPhaseTimings::degen_gc_final_update_refs_rebuild_freeset); + size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); + ShenandoahHeapLocker locker(lock()); + size_t young_cset_regions, old_cset_regions; + _free_set->prepare_to_rebuild(young_cset_regions, old_cset_regions); + + if (mode()->is_generational()) { + assert(verify_generation_usage(true, old_generation()->used_regions(), + old_generation()->used(), old_generation()->get_humongous_waste(), + true, young_generation()->used_regions(), + young_generation()->used(), young_generation()->get_humongous_waste()), + "Generation accounts are inaccurate"); + + // The computation of bytes_of_allocation_runway_before_gc_trigger is quite conservative so consider all of this + // available for transfer to old. Note that transfer of humongous regions does not impact available. + size_t allocation_runway = young_generation()->heuristics()->bytes_of_allocation_runway_before_gc_trigger(young_cset_regions); + adjust_generation_sizes_for_next_cycle(allocation_runway, young_cset_regions, old_cset_regions); + + // Total old_available may have been expanded to hold anticipated promotions. We trigger if the fragmented available + // memory represents more than 16 regions worth of data. Note that fragmentation may increase when we promote regular + // regions in place when many of these regular regions have an abundant amount of available memory within them. Fragmentation + // will decrease as promote-by-copy consumes the available memory within these partially consumed regions. + // + // We consider old-gen to have excessive fragmentation if more than 12.5% of old-gen is free memory that resides + // within partially consumed regions of memory. + } + // Rebuild free set based on adjusted generation sizes. + _free_set->rebuild(young_cset_regions, old_cset_regions); + + if (mode()->is_generational()) { + size_t old_available = old_generation()->available(); + size_t old_unaffiliated_available = old_generation()->free_unaffiliated_regions() * region_size_bytes; + size_t old_fragmented_available; + assert(old_available >= old_unaffiliated_available, "unaffiliated available is a subset of total available"); + old_fragmented_available = old_available - old_unaffiliated_available; + + size_t old_capacity = old_generation()->max_capacity(); + size_t heap_capacity = capacity(); + if ((old_capacity > heap_capacity / 8) && (old_fragmented_available > old_capacity / 8)) { + ((ShenandoahOldHeuristics *) old_generation()->heuristics())->trigger_old_is_fragmented(); + } + + size_t old_used = old_generation()->used() + old_generation()->get_humongous_waste(); + size_t trigger_threshold = old_generation()->usage_trigger_threshold(); + // Detects unsigned arithmetic underflow + assert(old_used < ShenandoahHeap::heap()->capacity(), "Old used must be less than heap capacity"); + + if (old_used > trigger_threshold) { + ((ShenandoahOldHeuristics *) old_generation()->heuristics())->trigger_old_has_grown(); + } } } @@ -3187,6 +3346,57 @@ void ShenandoahGenerationRegionClosure::heap_region_do(Shenandoa _cl->heap_region_do(region); } +bool ShenandoahHeap::verify_generation_usage(bool verify_old, size_t old_regions, size_t old_bytes, size_t old_waste, + bool verify_young, size_t young_regions, size_t young_bytes, size_t young_waste) { + size_t tally_old_regions = 0; + size_t tally_old_bytes = 0; + size_t tally_old_waste = 0; + size_t tally_young_regions = 0; + size_t tally_young_bytes = 0; + size_t tally_young_waste = 0; + + shenandoah_assert_heaplocked_or_safepoint(); + for (size_t i = 0; i < num_regions(); i++) { + ShenandoahHeapRegion* r = get_region(i); + if (r->is_old()) { + tally_old_regions++; + tally_old_bytes += r->used(); + if (r->is_humongous()) { + ShenandoahHeapRegion* start = r->humongous_start_region(); + HeapWord* obj_addr = start->bottom(); + oop obj = cast_to_oop(obj_addr); + size_t word_size = obj->size(); + HeapWord* end_addr = obj_addr + word_size; + if (end_addr <= r->end()) { + tally_old_waste += (r->end() - end_addr) * HeapWordSize; + } + } + } else if (r->is_young()) { + tally_young_regions++; + tally_young_bytes += r->used(); + if (r->is_humongous()) { + ShenandoahHeapRegion* start = r->humongous_start_region(); + HeapWord* obj_addr = start->bottom(); + oop obj = cast_to_oop(obj_addr); + size_t word_size = obj->size(); + HeapWord* end_addr = obj_addr + word_size; + if (end_addr <= r->end()) { + tally_young_waste += (r->end() - end_addr) * HeapWordSize; + } + } + } + } + if (verify_young && + ((young_regions != tally_young_regions) || (young_bytes != tally_young_bytes) || (young_waste != tally_young_waste))) { + return false; + } else if (verify_old && + ((old_regions != tally_old_regions) || (old_bytes != tally_old_bytes) || (old_waste != tally_old_waste))) { + return false; + } else { + return true; + } +} + ShenandoahGeneration* ShenandoahHeap::generation_for(ShenandoahAffiliation affiliation) const { if (!mode()->is_generational()) { return global_generation(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp index 105835e5728b0..368ae2a1ab323 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.hpp @@ -205,6 +205,9 @@ class ShenandoahHeap : public CollectedHeap { void prepare_for_verify() override; void verify(VerifyOption vo) override; + bool verify_generation_usage(bool verify_old, size_t old_regions, size_t old_bytes, size_t old_waste, + bool verify_young, size_t young_regions, size_t young_bytes, size_t young_waste); + // WhiteBox testing support. bool supports_concurrent_gc_breakpoints() const override { return true; @@ -215,6 +218,14 @@ class ShenandoahHeap : public CollectedHeap { private: size_t _initial_size; size_t _minimum_size; + size_t _promotion_potential; + size_t _promotion_in_place_potential; + size_t _pad_for_promote_in_place; // bytes of filler + size_t _promotable_humongous_regions; + size_t _promotable_humongous_usage; + size_t _regular_regions_promoted_in_place; + size_t _regular_usage_promoted_in_place; + volatile size_t _soft_max_size; shenandoah_padding(0); volatile size_t _committed; @@ -284,6 +295,8 @@ class ShenandoahHeap : public CollectedHeap { void heap_region_iterate(ShenandoahHeapRegionClosure* blk) const; void parallel_heap_region_iterate(ShenandoahHeapRegionClosure* blk) const; + inline ShenandoahMmuTracker* const mmu_tracker() { return &_mmu_tracker; }; + // ---------- GC state machinery // // GC state describes the important parts of collector state, that may be @@ -332,31 +345,8 @@ class ShenandoahHeap : public CollectedHeap { ShenandoahSharedFlag _progress_last_gc; ShenandoahSharedFlag _concurrent_strong_root_in_progress; - // _alloc_supplement_reserve is a supplemental budget for new_memory allocations. During evacuation and update-references, - // mutator allocation requests are "authorized" iff young_gen->available() plus _alloc_supplement_reserve minus - // _young_evac_reserve is greater than request size. The values of _alloc_supplement_reserve and _young_evac_reserve - // are zero except during evacuation and update-reference phases of GC. Both of these values are established at - // the start of evacuation, and they remain constant throughout the duration of these two phases of GC. Since these - // two values are constant throughout each GC phases, we introduce a new service into ShenandoahGeneration. This service - // provides adjusted_available() based on an adjusted capacity. At the start of evacuation, we adjust young capacity by - // adding the amount to be borrowed from old-gen and subtracting the _young_evac_reserve, we adjust old capacity by - // subtracting the amount to be loaned to young-gen. - // - // We always use adjusted capacities to determine permission to allocate within young and to promote into old. Note - // that adjusted capacities equal traditional capacities except during evacuation and update refs. - // - // During evacuation, we assure that _old_evac_expended does not exceed _old_evac_reserve. - // - // At the end of update references, we perform the following bookkeeping activities: - // - // 1. Unadjust the capacity within young-gen and old-gen to undo the effects of borrowing memory from old-gen. Note that - // the entirety of the collection set is now available, so allocation capacity naturally increase at this time. - // 2. Clear (reset to zero) _alloc_supplement_reserve, _young_evac_reserve, _old_evac_reserve, and _promoted_reserve - // - // _young_evac_reserve and _old_evac_reserve are only non-zero during evacuation and update-references. - // - // Allocation of old GCLABs assures that _old_evac_expended + request-size < _old_evac_reserved. If the allocation - // is authorized, increment _old_evac_expended by request size. This allocation ignores old_gen->available(). + // TODO: Revisit the following comment. It may not accurately represent the true behavior when evacuations fail due to + // difficulty finding memory to hold evacuated objects. // // Note that the typical total expenditure on evacuation is less than the associated evacuation reserve because we generally // reserve ShenandoahEvacWaste (> 1.0) times the anticipated evacuation need. In the case that there is an excessive amount @@ -364,10 +354,15 @@ class ShenandoahHeap : public CollectedHeap { // effort. If this happens, the requesting thread blocks until some other thread manages to evacuate the offending object. // Only after "all" threads fail to evacuate an object do we consider the evacuation effort to have failed. - intptr_t _alloc_supplement_reserve; // Bytes reserved for young allocations during evac and update refs + // How many full-gc cycles have been completed? + volatile size_t _completed_fullgc_cycles; + size_t _promoted_reserve; // Bytes reserved within old-gen to hold the results of promotion volatile size_t _promoted_expended; // Bytes of old-gen memory expended on promotions + // Allocation of old GCLABs (aka PLABs) assures that _old_evac_expended + request-size < _old_evac_reserved. If the allocation + // is authorized, increment _old_evac_expended by request size. This allocation ignores old_gen->available(). + size_t _old_evac_reserve; // Bytes reserved within old-gen to hold evacuated objects from old-gen collection set volatile size_t _old_evac_expended; // Bytes of old-gen memory expended on old-gen evacuations @@ -397,7 +392,6 @@ class ShenandoahHeap : public CollectedHeap { void set_gc_state_mask(uint mask, bool value); public: - char gc_state() const; static address gc_state_addr(); @@ -415,6 +409,7 @@ class ShenandoahHeap : public CollectedHeap { void set_prepare_for_old_mark_in_progress(bool cond); void set_aging_cycle(bool cond); + inline bool is_stable() const; inline bool is_idle() const; inline bool has_evacuation_reserve_quantities() const; @@ -441,9 +436,31 @@ class ShenandoahHeap : public CollectedHeap { inline void set_previous_promotion(size_t promoted_bytes); inline size_t get_previous_promotion() const; + inline void clear_promotion_potential() { _promotion_potential = 0; }; + inline void set_promotion_potential(size_t val) { _promotion_potential = val; }; + inline size_t get_promotion_potential() { return _promotion_potential; }; + + inline void clear_promotion_in_place_potential() { _promotion_in_place_potential = 0; }; + inline void set_promotion_in_place_potential(size_t val) { _promotion_in_place_potential = val; }; + inline size_t get_promotion_in_place_potential() { return _promotion_in_place_potential; }; + + inline void set_pad_for_promote_in_place(size_t pad) { _pad_for_promote_in_place = pad; } + inline size_t get_pad_for_promote_in_place() { return _pad_for_promote_in_place; } + + inline void reserve_promotable_humongous_regions(size_t region_count) { _promotable_humongous_regions = region_count; } + inline void reserve_promotable_humongous_usage(size_t bytes) { _promotable_humongous_usage = bytes; } + inline void reserve_promotable_regular_regions(size_t region_count) { _regular_regions_promoted_in_place = region_count; } + inline void reserve_promotable_regular_usage(size_t used_bytes) { _regular_usage_promoted_in_place = used_bytes; } + + inline size_t get_promotable_humongous_regions() { return _promotable_humongous_regions; } + inline size_t get_promotable_humongous_usage() { return _promotable_humongous_usage; } + inline size_t get_regular_regions_promoted_in_place() { return _regular_regions_promoted_in_place; } + inline size_t get_regular_usage_promoted_in_place() { return _regular_usage_promoted_in_place; } + // Returns previous value inline size_t set_promoted_reserve(size_t new_val); inline size_t get_promoted_reserve() const; + inline void augment_promo_reserve(size_t increment); inline void reset_promoted_expended(); inline size_t expend_promoted(size_t increment); @@ -453,6 +470,7 @@ class ShenandoahHeap : public CollectedHeap { // Returns previous value inline size_t set_old_evac_reserve(size_t new_val); inline size_t get_old_evac_reserve() const; + inline void augment_old_evac_reserve(size_t increment); inline void reset_old_evac_expended(); inline size_t expend_old_evac(size_t increment); @@ -462,11 +480,6 @@ class ShenandoahHeap : public CollectedHeap { inline size_t set_young_evac_reserve(size_t new_val); inline size_t get_young_evac_reserve() const; - // Returns previous value. This is a signed value because it is the amount borrowed minus the amount reserved for - // young-gen evacuation. In case we cannot borrow much, this value might be negative. - inline intptr_t set_alloc_supplement_reserve(intptr_t new_val); - inline intptr_t get_alloc_supplement_reserve() const; - private: void manage_satb_barrier(bool active); @@ -517,11 +530,11 @@ class ShenandoahHeap : public CollectedHeap { void update_heap_references(bool concurrent); // Final update region states void update_heap_region_states(bool concurrent); - void rebuild_free_set(bool concurrent); void rendezvous_threads(); void recycle_trash(); public: + void rebuild_free_set(bool concurrent); void notify_gc_progress() { _progress_last_gc.set(); } void notify_gc_no_progress() { _progress_last_gc.unset(); } @@ -698,6 +711,10 @@ class ShenandoahHeap : public CollectedHeap { // ---------- Allocation support // private: + // How many bytes to transfer between old and young after we have finished recycling collection set regions? + size_t _old_regions_surplus; + size_t _old_regions_deficit; + HeapWord* allocate_memory_under_lock(ShenandoahAllocRequest& request, bool& in_new_region, bool is_promotion); inline HeapWord* allocate_from_gclab(Thread* thread, size_t size); @@ -731,6 +748,12 @@ class ShenandoahHeap : public CollectedHeap { void set_young_lab_region_flags(); + inline void set_old_region_surplus(size_t surplus) { _old_regions_surplus = surplus; }; + inline void set_old_region_deficit(size_t deficit) { _old_regions_deficit = deficit; }; + + inline size_t get_old_region_surplus() { return _old_regions_surplus; }; + inline size_t get_old_region_deficit() { return _old_regions_deficit; }; + // ---------- Marking support // private: @@ -830,7 +853,7 @@ class ShenandoahHeap : public CollectedHeap { void cancel_old_gc(); bool is_old_gc_active(); void coalesce_and_fill_old_regions(); - bool adjust_generation_sizes(); + void adjust_generation_sizes_for_next_cycle(size_t old_xfer_limit, size_t young_cset_regions, size_t old_cset_regions); // ---------- Helper functions // diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp index d131d336e780a..d6dccf01f9daf 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.inline.hpp @@ -291,11 +291,12 @@ inline HeapWord* ShenandoahHeap::allocate_from_plab(Thread* thread, size_t size, PLAB* plab = ShenandoahThreadLocalData::plab(thread); HeapWord* obj; + if (plab == nullptr) { assert(!thread->is_Java_thread() && !thread->is_Worker_thread(), "Performance: thread should have PLAB: %s", thread->name()); // No PLABs in this thread, fallback to shared allocation return nullptr; - } else if (is_promotion && (plab->words_remaining() > 0) && !ShenandoahThreadLocalData::allow_plab_promotions(thread)) { + } else if (is_promotion && !ShenandoahThreadLocalData::allow_plab_promotions(thread)) { return nullptr; } // if plab->word_size() <= 0, thread's plab not yet initialized for this pass, so allow_plab_promotions() is not trustworthy @@ -382,7 +383,6 @@ inline oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, Shenandoah break; } case OLD_GENERATION: { - PLAB* plab = ShenandoahThreadLocalData::plab(thread); if (plab != nullptr) { has_plab = true; @@ -510,6 +510,7 @@ inline oop ShenandoahHeap::try_evacuate_object(oop p, Thread* thread, Shenandoah // For non-LAB allocations, we have no way to retract the allocation, and // have to explicitly overwrite the copy with the filler object. With that overwrite, // we have to keep the fwdptr initialized and pointing to our (stale) copy. + assert(size >= ShenandoahHeap::min_fill_size(), "previously allocated object known to be larger than min_size"); fill_with_object(copy, size); shenandoah_assert_correct(nullptr, copy_val); // For non-LAB allocations, the object has already been registered @@ -753,6 +754,14 @@ inline size_t ShenandoahHeap::get_old_evac_reserve() const { return _old_evac_reserve; } +inline void ShenandoahHeap::augment_old_evac_reserve(size_t increment) { + _old_evac_reserve += increment; +} + +inline void ShenandoahHeap::augment_promo_reserve(size_t increment) { + _promoted_reserve += increment; +} + inline void ShenandoahHeap::reset_old_evac_expended() { Atomic::store(&_old_evac_expended, (size_t) 0); } @@ -791,16 +800,6 @@ inline size_t ShenandoahHeap::get_young_evac_reserve() const { return _young_evac_reserve; } -inline intptr_t ShenandoahHeap::set_alloc_supplement_reserve(intptr_t new_val) { - intptr_t orig = _alloc_supplement_reserve; - _alloc_supplement_reserve = new_val; - return orig; -} - -inline intptr_t ShenandoahHeap::get_alloc_supplement_reserve() const { - return _alloc_supplement_reserve; -} - template inline void ShenandoahHeap::marked_object_iterate(ShenandoahHeapRegion* region, T* cl) { marked_object_iterate(region, cl, region->top()); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp index 299c2b1c593f2..941f4f1ec7311 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.cpp @@ -29,6 +29,7 @@ #include "gc/shared/space.inline.hpp" #include "gc/shared/tlab_globals.hpp" #include "gc/shenandoah/shenandoahCardTable.hpp" +#include "gc/shenandoah/shenandoahFreeSet.hpp" #include "gc/shenandoah/shenandoahHeapRegionSet.inline.hpp" #include "gc/shenandoah/shenandoahHeap.inline.hpp" #include "gc/shenandoah/shenandoahHeapRegion.hpp" @@ -102,7 +103,7 @@ void ShenandoahHeapRegion::make_regular_allocation(ShenandoahAffiliation affilia case _empty_uncommitted: do_commit(); case _empty_committed: - set_affiliation(affiliation); + assert(this->affiliation() == affiliation, "Region affiliation should already be established"); set_state(_regular); case _regular: case _pinned: @@ -122,7 +123,13 @@ void ShenandoahHeapRegion::make_young_maybe() { case _cset: case _humongous_start: case _humongous_cont: - set_affiliation(YOUNG_GENERATION); + if (affiliation() != YOUNG_GENERATION) { + if (is_old()) { + ShenandoahHeap::heap()->old_generation()->decrement_affiliated_region_count(); + } + set_affiliation(YOUNG_GENERATION); + ShenandoahHeap::heap()->young_generation()->increment_affiliated_region_count(); + } return; case _pinned_cset: case _regular: @@ -175,6 +182,7 @@ void ShenandoahHeapRegion::make_humongous_start() { void ShenandoahHeapRegion::make_humongous_start_bypass(ShenandoahAffiliation affiliation) { shenandoah_assert_heaplocked(); assert (ShenandoahHeap::heap()->is_full_gc_in_progress(), "only for full GC"); + // Don't bother to account for affiliated regions during Full GC. We recompute totals at end. set_affiliation(affiliation); reset_age(); switch (_state) { @@ -207,6 +215,7 @@ void ShenandoahHeapRegion::make_humongous_cont_bypass(ShenandoahAffiliation affi shenandoah_assert_heaplocked(); assert (ShenandoahHeap::heap()->is_full_gc_in_progress(), "only for full GC"); set_affiliation(affiliation); + // Don't bother to account for affiliated regions during Full GC. We recompute totals at end. reset_age(); switch (_state) { case _empty_committed: @@ -469,6 +478,7 @@ bool ShenandoahHeapRegion::oop_fill_and_coalesce_without_cancel() { HeapWord* next_marked_obj = marking_context->get_next_marked_addr(obj_addr, t); assert(next_marked_obj <= t, "next marked object cannot exceed top"); size_t fill_size = next_marked_obj - obj_addr; + assert(fill_size >= ShenandoahHeap::min_fill_size(), "previously allocated objects known to be larger than min_size"); ShenandoahHeap::fill_with_object(obj_addr, fill_size); heap->card_scan()->coalesce_objects(obj_addr, fill_size); obj_addr = next_marked_obj; @@ -514,6 +524,7 @@ bool ShenandoahHeapRegion::oop_fill_and_coalesce() { HeapWord* next_marked_obj = marking_context->get_next_marked_addr(obj_addr, t); assert(next_marked_obj <= t, "next marked object cannot exceed top"); size_t fill_size = next_marked_obj - obj_addr; + assert(fill_size >= ShenandoahHeap::min_fill_size(), "previously allocated object known to be larger than min_size"); ShenandoahHeap::fill_with_object(obj_addr, fill_size); heap->card_scan()->coalesce_objects(obj_addr, fill_size); obj_addr = next_marked_obj; @@ -565,8 +576,8 @@ void ShenandoahHeapRegion::global_oop_iterate_objects_and_fill_dead(OopIterateCl HeapWord* next_marked_obj = marking_context->get_next_marked_addr(obj_addr, t); assert(next_marked_obj <= t, "next marked object cannot exceed top"); size_t fill_size = next_marked_obj - obj_addr; + assert(fill_size >= ShenandoahHeap::min_fill_size(), "previously allocated objects known to be larger than min_size"); ShenandoahHeap::fill_with_object(obj_addr, fill_size); - // coalesce_objects() unregisters all but first object subsumed within coalesced range. rem_set_scanner->coalesce_objects(obj_addr, fill_size); obj_addr = next_marked_obj; @@ -670,8 +681,8 @@ void ShenandoahHeapRegion::recycle() { set_update_watermark(bottom()); make_empty(); + ShenandoahHeap::heap()->generation_for(affiliation())->decrement_affiliated_region_count(); set_affiliation(FREE); - if (ZapUnusedHeapArea) { SpaceMangler::mangle_region(MemRegion(bottom(), end())); } @@ -957,40 +968,22 @@ void ShenandoahHeapRegion::set_affiliation(ShenandoahAffiliation new_affiliation } if (!heap->mode()->is_generational()) { + log_trace(gc)("Changing affiliation of region %zu from %s to %s", + index(), affiliation_name(), shenandoah_affiliation_name(new_affiliation)); heap->set_affiliation(this, new_affiliation); return; } - log_trace(gc)("Changing affiliation of region %zu from %s to %s", - index(), shenandoah_affiliation_name(region_affiliation), shenandoah_affiliation_name(new_affiliation)); - - if (region_affiliation == ShenandoahAffiliation::YOUNG_GENERATION) { - heap->young_generation()->decrement_affiliated_region_count(); - } else if (region_affiliation == ShenandoahAffiliation::OLD_GENERATION) { - heap->old_generation()->decrement_affiliated_region_count(); - } - - size_t regions; switch (new_affiliation) { case FREE: assert(!has_live(), "Free region should not have live data"); break; case YOUNG_GENERATION: reset_age(); - regions = heap->young_generation()->increment_affiliated_region_count(); - // During Full GC, we allow temporary violation of this requirement. We enforce that this condition is - // restored upon completion of Full GC. - assert(heap->is_full_gc_in_progress() || - (regions * ShenandoahHeapRegion::region_size_bytes() <= heap->young_generation()->adjusted_capacity()), - "Number of young regions cannot exceed adjusted capacity"); break; case OLD_GENERATION: - regions = heap->old_generation()->increment_affiliated_region_count(); - // During Full GC, we allow temporary violation of this requirement. We enforce that this condition is - // restored upon completion of Full GC. - assert(heap->is_full_gc_in_progress() || - (regions * ShenandoahHeapRegion::region_size_bytes() <= heap->old_generation()->adjusted_capacity()), - "Number of old regions cannot exceed adjusted capacity"); + // TODO: should we reset_age() for OLD as well? Examine invocations of set_affiliation(). Some contexts redundantly + // invoke reset_age(). break; default: ShouldNotReachHere(); @@ -999,8 +992,92 @@ void ShenandoahHeapRegion::set_affiliation(ShenandoahAffiliation new_affiliation heap->set_affiliation(this, new_affiliation); } -// Returns number of regions promoted, or zero if we choose not to promote. -size_t ShenandoahHeapRegion::promote_humongous() { +// When we promote a region in place, we can continue to use the established marking context to guide subsequent remembered +// set scans of this region's content. The region will be coalesced and filled prior to the next old-gen marking effort. +// We identify the entirety of the region as DIRTY to force the next remembered set scan to identify the "interesting poitners" +// contained herein. +void ShenandoahHeapRegion::promote_in_place() { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + ShenandoahMarkingContext* marking_context = heap->marking_context(); + HeapWord* tams = marking_context->top_at_mark_start(this); + assert(heap->active_generation()->is_mark_complete(), "sanity"); + assert(is_young(), "Only young regions can be promoted"); + assert(is_regular(), "Use different service to promote humongous regions"); + assert(age() >= InitialTenuringThreshold, "Only promote regions that are sufficiently aged"); + + ShenandoahOldGeneration* old_gen = heap->old_generation(); + ShenandoahYoungGeneration* young_gen = heap->young_generation(); + size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); + + { + ShenandoahHeapLocker locker(heap->lock()); + + HeapWord* update_watermark = get_update_watermark(); + + // Now that this region is affiliated with old, we can allow it to receive allocations, though it may not be in the + // is_collector_free range. + restore_top_before_promote(); + + size_t region_capacity = free(); + size_t region_used = used(); + + // The update_watermark was likely established while we had the artificially high value of top. Make it sane now. + assert(update_watermark >= top(), "original top cannot exceed preserved update_watermark"); + set_update_watermark(top()); + + // Unconditionally transfer one region from young to old to represent the newly promoted region. + // This expands old and shrinks new by the size of one region. Strictly, we do not "need" to expand old + // if there are already enough unaffiliated regions in old to account for this newly promoted region. + // However, if we do not transfer the capacities, we end up reducing the amount of memory that would have + // otherwise been available to hold old evacuations, because old available is max_capacity - used and now + // we would be trading a fully empty region for a partially used region. + + young_gen->decrease_used(region_used); + young_gen->decrement_affiliated_region_count(); + + // transfer_to_old() increases capacity of old and decreases capacity of young + heap->generation_sizer()->force_transfer_to_old(1); + set_affiliation(OLD_GENERATION); + + old_gen->increment_affiliated_region_count(); + old_gen->increase_used(region_used); + + // add_old_collector_free_region() increases promoted_reserve() if available space exceeds PLAB::min_size() + heap->free_set()->add_old_collector_free_region(this); + } + + assert(top() == tams, "Cannot promote regions in place if top has advanced beyond TAMS"); + + // Since this region may have served previously as OLD, it may hold obsolete object range info. + heap->card_scan()->reset_object_range(bottom(), end()); + heap->card_scan()->mark_range_as_dirty(bottom(), top() - bottom()); + + // TODO: use an existing coalesce-and-fill function rather than + // replicating the code here. + HeapWord* obj_addr = bottom(); + while (obj_addr < tams) { + oop obj = cast_to_oop(obj_addr); + if (marking_context->is_marked(obj)) { + assert(obj->klass() != NULL, "klass should not be NULL"); + // This thread is responsible for registering all objects in this region. No need for lock. + heap->card_scan()->register_object_without_lock(obj_addr); + obj_addr += obj->size(); + } else { + HeapWord* next_marked_obj = marking_context->get_next_marked_addr(obj_addr, tams); + assert(next_marked_obj <= tams, "next marked object cannot exceed tams"); + size_t fill_size = next_marked_obj - obj_addr; + assert(fill_size >= ShenandoahHeap::min_fill_size(), "previously allocated objects known to be larger than min_size"); + ShenandoahHeap::fill_with_object(obj_addr, fill_size); + heap->card_scan()->register_object_without_lock(obj_addr); + obj_addr = next_marked_obj; + } + } + + // We do not need to scan above TAMS because top equals tams + assert(obj_addr == tams, "Expect loop to terminate when obj_addr equals tams"); +} + +void ShenandoahHeapRegion::promote_humongous() { ShenandoahHeap* heap = ShenandoahHeap::heap(); ShenandoahMarkingContext* marking_context = heap->marking_context(); assert(heap->active_generation()->is_mark_complete(), "sanity"); @@ -1020,51 +1097,40 @@ size_t ShenandoahHeapRegion::promote_humongous() { // it becomes garbage. Better to not make this change until sizes of young-gen and old-gen are completely // adaptive, as leaving primitive arrays in young-gen might be perceived as an "astonishing result" by someone // has carefully analyzed the required sizes of an application's young-gen and old-gen. - - size_t spanned_regions = ShenandoahHeapRegion::required_regions(obj->size() * HeapWordSize); + size_t used_bytes = obj->size() * HeapWordSize; + size_t spanned_regions = ShenandoahHeapRegion::required_regions(used_bytes); + size_t humongous_waste = spanned_regions * ShenandoahHeapRegion::region_size_bytes() - obj->size() * HeapWordSize; size_t index_limit = index() + spanned_regions; - { // We need to grab the heap lock in order to avoid a race when changing the affiliations of spanned_regions from // young to old. ShenandoahHeapLocker locker(heap->lock()); - size_t available_old_regions = old_generation->adjusted_unaffiliated_regions(); - if (spanned_regions <= available_old_regions) { - log_debug(gc)("promoting humongous region " SIZE_FORMAT ", spanning " SIZE_FORMAT, index(), spanned_regions); - - // For this region and each humongous continuation region spanned by this humongous object, change - // affiliation to OLD_GENERATION and adjust the generation-use tallies. The remnant of memory - // in the last humongous region that is not spanned by obj is currently not used. - for (size_t i = index(); i < index_limit; i++) { - ShenandoahHeapRegion* r = heap->get_region(i); - log_debug(gc)("promoting humongous region " SIZE_FORMAT ", from " PTR_FORMAT " to " PTR_FORMAT, - r->index(), p2i(r->bottom()), p2i(r->top())); - // We mark the entire humongous object's range as dirty after loop terminates, so no need to dirty the range here - old_generation->increase_used(r->used()); - young_generation->decrease_used(r->used()); - r->set_affiliation(OLD_GENERATION); - } - ShenandoahHeapRegion* tail = heap->get_region(index_limit - 1); - size_t waste = tail->free(); - if (waste != 0) { - old_generation->increase_humongous_waste(waste); - young_generation->decrease_humongous_waste(waste); - } - // Then fall through to finish the promotion after releasing the heap lock. - } else { - // There are not enough available old regions to promote this humongous region at this time, so defer promotion. - // TODO: Consider allowing the promotion now, with the expectation that we can resize and/or collect OLD - // momentarily to address the transient violation of budgets. Some problems that need to be addressed in order - // to allow transient violation of capacity budgets are: - // 1. Various size_t subtractions assume usage is less than capacity, and thus assume there will be no - // arithmetic underflow when we subtract usage from capacity. The results of such size_t subtractions - // would need to be guarded and special handling provided. - // 2. ShenandoahVerifier enforces that usage is less than capacity. If we are going to relax this constraint, - // we need to think about what conditions allow the constraint to be violated and document and implement the - // changes. - return 0; + // We promote humongous objects unconditionally, without checking for availability. We adjust + // usage totals, including humongous waste, after evacuation is done. + log_debug(gc)("promoting humongous region " SIZE_FORMAT ", spanning " SIZE_FORMAT, index(), spanned_regions); + + young_generation->decrease_used(used_bytes); + young_generation->decrease_humongous_waste(humongous_waste); + young_generation->decrease_affiliated_region_count(spanned_regions); + + // transfer_to_old() increases capacity of old and decreases capacity of young + heap->generation_sizer()->force_transfer_to_old(spanned_regions); + + // For this region and each humongous continuation region spanned by this humongous object, change + // affiliation to OLD_GENERATION and adjust the generation-use tallies. The remnant of memory + // in the last humongous region that is not spanned by obj is currently not used. + for (size_t i = index(); i < index_limit; i++) { + ShenandoahHeapRegion* r = heap->get_region(i); + log_debug(gc)("promoting humongous region " SIZE_FORMAT ", from " PTR_FORMAT " to " PTR_FORMAT, + r->index(), p2i(r->bottom()), p2i(r->top())); + // We mark the entire humongous object's range as dirty after loop terminates, so no need to dirty the range here + r->set_affiliation(OLD_GENERATION); } + + old_generation->increase_affiliated_region_count(spanned_regions); + old_generation->increase_used(used_bytes); + old_generation->increase_humongous_waste(humongous_waste); } // Since this region may have served previously as OLD, it may hold obsolete object range info. @@ -1082,7 +1148,6 @@ size_t ShenandoahHeapRegion::promote_humongous() { index(), p2i(bottom()), p2i(bottom() + obj->size())); heap->card_scan()->mark_range_as_dirty(bottom(), obj->size()); } - return index_limit - index(); } void ShenandoahHeapRegion::decrement_humongous_waste() const { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp index 269f0fd239be2..306cff58508e9 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.hpp @@ -242,6 +242,8 @@ class ShenandoahHeapRegion { HeapWord* _new_top; double _empty_time; + HeapWord* _top_before_promoted; + // Seldom updated fields RegionState _state; HeapWord* _coalesce_and_fill_boundary; // for old regions not selected as collection set candidates. @@ -350,6 +352,11 @@ class ShenandoahHeapRegion { return _index; } + inline void save_top_before_promote(); + inline HeapWord* get_top_before_promote() const { return _top_before_promoted; } + inline void restore_top_before_promote(); + inline size_t garbage_before_padded_for_promote() const; + // Allocation (return nullptr if full) inline HeapWord* allocate_aligned(size_t word_size, ShenandoahAllocRequest &req, size_t alignment_in_words); @@ -429,6 +436,7 @@ class ShenandoahHeapRegion { size_t capacity() const { return byte_size(bottom(), end()); } size_t used() const { return byte_size(bottom(), top()); } + size_t used_before_promote() const { return byte_size(bottom(), get_top_before_promote()); } size_t free() const { return byte_size(top(), end()); } // Does this region contain this address? @@ -457,8 +465,9 @@ class ShenandoahHeapRegion { void decrement_age() { if (_age-- == 0) { _age = 0; } } void reset_age() { _age = 0; } - // Sets all remembered set cards to dirty. Returns the number of regions spanned by the associated humongous object. - size_t promote_humongous(); + // Register all objects. Set all remembered set cards to dirty. + void promote_humongous(); + void promote_in_place(); private: void decrement_humongous_waste() const; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp index 4ef7f36ab1df9..c5823b11ddd6d 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp @@ -82,6 +82,7 @@ HeapWord* ShenandoahHeapRegion::allocate_aligned(size_t size, ShenandoahAllocReq if (size >= req.min_size()) { // Even if req.min_size() is not a multiple of card size, we know that size is. if (pad_words > 0) { + assert(pad_words >= ShenandoahHeap::min_fill_size(), "pad_words expanded above to meet size constraint"); ShenandoahHeap::fill_with_object(orig_top, pad_words); ShenandoahHeap::heap()->card_scan()->register_object(orig_top); } @@ -190,6 +191,17 @@ inline size_t ShenandoahHeapRegion::garbage() const { return result; } +inline size_t ShenandoahHeapRegion::garbage_before_padded_for_promote() const { + size_t used_before_promote = byte_size(bottom(), get_top_before_promote()); + assert(get_top_before_promote() != nullptr, "top before promote should not equal null"); + assert(used_before_promote >= get_live_data_bytes(), + "Live Data must be a subset of used before promotion live: " SIZE_FORMAT " used: " SIZE_FORMAT, + get_live_data_bytes(), used_before_promote); + size_t result = used_before_promote - get_live_data_bytes(); + return result; + +} + inline HeapWord* ShenandoahHeapRegion::get_update_watermark() const { HeapWord* watermark = Atomic::load_acquire(&_update_watermark); assert(bottom() <= watermark && watermark <= top(), "within bounds"); @@ -240,4 +252,17 @@ inline bool ShenandoahHeapRegion::is_affiliated() const { return affiliation() != FREE; } +inline void ShenandoahHeapRegion::save_top_before_promote() { + _top_before_promoted = _top; +} + +inline void ShenandoahHeapRegion::restore_top_before_promote() { + _top = _top_before_promoted; +#ifdef ASSERT + _top_before_promoted = nullptr; +#endif + } + + + #endif // SHARE_GC_SHENANDOAH_SHENANDOAHHEAPREGION_INLINE_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahInitLogger.cpp b/src/hotspot/share/gc/shenandoah/shenandoahInitLogger.cpp index e9268063a563c..76cc19fd47d42 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahInitLogger.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahInitLogger.cpp @@ -47,14 +47,14 @@ void ShenandoahInitLogger::print_heap() { log_info(gc, init)("Heuristics: %s", heap->global_generation()->heuristics()->name()); } else { log_info(gc, init)("Young Heuristics: %s", heap->young_generation()->heuristics()->name()); - log_info(gc, init)("Young Generation Initial Size: " SIZE_FORMAT "%s", + log_info(gc, init)("Young Generation Soft Size: " SIZE_FORMAT "%s", byte_size_in_proper_unit(heap->young_generation()->soft_max_capacity()), proper_unit_for_byte_size(heap->young_generation()->soft_max_capacity())); log_info(gc, init)("Young Generation Max: " SIZE_FORMAT "%s", byte_size_in_proper_unit(heap->young_generation()->max_capacity()), proper_unit_for_byte_size(heap->young_generation()->max_capacity())); log_info(gc, init)("Old Heuristics: %s", heap->old_generation()->heuristics()->name()); - log_info(gc, init)("Old Generation Initial Size: " SIZE_FORMAT "%s", + log_info(gc, init)("Old Generation Soft Size: " SIZE_FORMAT "%s", byte_size_in_proper_unit(heap->old_generation()->soft_max_capacity()), proper_unit_for_byte_size(heap->old_generation()->soft_max_capacity())); log_info(gc, init)("Old Generation Max: " SIZE_FORMAT "%s", diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp index b9443f129239a..a7688679f2913 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.cpp @@ -32,7 +32,6 @@ #include "runtime/os.hpp" #include "runtime/task.hpp" - class ShenandoahMmuTask : public PeriodicTask { ShenandoahMmuTracker* _mmu_tracker; public: @@ -53,66 +52,134 @@ class ThreadTimeAccumulator : public ThreadClosure { } }; -double ShenandoahMmuTracker::gc_thread_time_seconds() { +ShenandoahMmuTracker::ShenandoahMmuTracker() : + _most_recent_timestamp(0.0), + _most_recent_gc_time(0.0), + _most_recent_gcu(0.0), + _most_recent_mutator_time(0.0), + _most_recent_mu(0.0), + _most_recent_periodic_time_stamp(0.0), + _most_recent_periodic_gc_time(0.0), + _most_recent_periodic_mutator_time(0.0), + _mmu_periodic_task(new ShenandoahMmuTask(this)) { +} + +ShenandoahMmuTracker::~ShenandoahMmuTracker() { + _mmu_periodic_task->disenroll(); + delete _mmu_periodic_task; +} + +void ShenandoahMmuTracker::fetch_cpu_times(double &gc_time, double &mutator_time) { ThreadTimeAccumulator cl; // We include only the gc threads because those are the only threads // we are responsible for. ShenandoahHeap::heap()->gc_threads_do(&cl); - return double(cl.total_time) / NANOSECS_PER_SEC; -} + double most_recent_gc_thread_time = double(cl.total_time) / NANOSECS_PER_SEC; + gc_time = most_recent_gc_thread_time; -double ShenandoahMmuTracker::process_time_seconds() { double process_real_time(0.0), process_user_time(0.0), process_system_time(0.0); bool valid = os::getTimesSecs(&process_real_time, &process_user_time, &process_system_time); - if (valid) { - return process_user_time + process_system_time; + assert(valid, "don't know why this would not be valid"); + mutator_time =(process_user_time + process_system_time) - most_recent_gc_thread_time; +} + +void ShenandoahMmuTracker::update_utilization(ShenandoahGeneration* generation, size_t gcid, const char *msg) { + double current = os::elapsedTime(); + _most_recent_gcid = gcid; + _most_recent_is_full = false; + + if (gcid == 0) { + fetch_cpu_times(_most_recent_gc_time, _most_recent_mutator_time); + + _most_recent_timestamp = current; + } else { + double gc_cycle_period = current - _most_recent_timestamp; + _most_recent_timestamp = current; + + double gc_thread_time, mutator_thread_time; + fetch_cpu_times(gc_thread_time, mutator_thread_time); + double gc_time = gc_thread_time - _most_recent_gc_time; + _most_recent_gc_time = gc_thread_time; + _most_recent_gcu = gc_time / (_active_processors * gc_cycle_period); + double mutator_time = mutator_thread_time - _most_recent_mutator_time; + _most_recent_mutator_time = mutator_thread_time; + _most_recent_mu = mutator_time / (_active_processors * gc_cycle_period); + log_info(gc, ergo)("At end of %s: GCU: %.1f%%, MU: %.1f%% during period of %.3fs", + msg, _most_recent_gcu * 100, _most_recent_mu * 100, gc_cycle_period); } - return 0.0; } -ShenandoahMmuTracker::ShenandoahMmuTracker() : - _generational_reference_time_s(0.0), - _process_reference_time_s(0.0), - _collector_reference_time_s(0.0), - _mmu_periodic_task(new ShenandoahMmuTask(this)), - _mmu_average(10, ShenandoahAdaptiveDecayFactor) { +void ShenandoahMmuTracker::record_young(ShenandoahGeneration* generation, size_t gcid) { + update_utilization(generation, gcid, "Concurrent Young GC"); } -ShenandoahMmuTracker::~ShenandoahMmuTracker() { - _mmu_periodic_task->disenroll(); - delete _mmu_periodic_task; +void ShenandoahMmuTracker::record_bootstrap(ShenandoahGeneration* generation, size_t gcid, bool candidates_for_mixed) { + // Not likely that this will represent an "ideal" GCU, but doesn't hurt to try + update_utilization(generation, gcid, "Bootstrap Old GC"); +} + +void ShenandoahMmuTracker::record_old_marking_increment(ShenandoahGeneration* generation, size_t gcid, bool old_marking_done, + bool has_old_candidates) { + // No special processing for old marking + double now = os::elapsedTime(); + double duration = now - _most_recent_timestamp; + + double gc_time, mutator_time; + fetch_cpu_times(gc_time, mutator_time); + double gcu = (gc_time - _most_recent_gc_time) / duration; + double mu = (mutator_time - _most_recent_mutator_time) / duration; + log_info(gc, ergo)("At end of %s: GCU: %.1f%%, MU: %.1f%% for duration %.3fs (totals to be subsumed in next gc report)", + old_marking_done? "last OLD marking increment": "OLD marking increment", + gcu * 100, mu * 100, duration); +} + +void ShenandoahMmuTracker::record_mixed(ShenandoahGeneration* generation, size_t gcid, bool is_mixed_done) { + update_utilization(generation, gcid, "Mixed Concurrent GC"); +} + +void ShenandoahMmuTracker::record_degenerated(ShenandoahGeneration* generation, + size_t gcid, bool is_old_bootstrap, bool is_mixed_done) { + if ((gcid == _most_recent_gcid) && _most_recent_is_full) { + // Do nothing. This is a redundant recording for the full gc that just completed. + // TODO: avoid making the call to record_degenerated() in the case that this degenerated upgraded to full gc. + } else if (is_old_bootstrap) { + update_utilization(generation, gcid, "Degenerated Bootstrap Old GC"); + } else { + update_utilization(generation, gcid, "Degenerated Young GC"); + } } -void ShenandoahMmuTracker::record(ShenandoahGeneration* generation) { - shenandoah_assert_control_or_vm_thread(); - double collector_time_s = gc_thread_time_seconds(); - double elapsed_gc_time_s = collector_time_s - _generational_reference_time_s; - generation->add_collection_time(elapsed_gc_time_s); - _generational_reference_time_s = collector_time_s; +void ShenandoahMmuTracker::record_full(ShenandoahGeneration* generation, size_t gcid) { + update_utilization(generation, gcid, "Full GC"); + _most_recent_is_full = true; } void ShenandoahMmuTracker::report() { // This is only called by the periodic thread. - double process_time_s = process_time_seconds(); - double elapsed_process_time_s = process_time_s - _process_reference_time_s; - if (elapsed_process_time_s <= 0.01) { - // No cpu time for this interval? - return; - } + double current = os::elapsedTime(); + double time_delta = current - _most_recent_periodic_time_stamp; + _most_recent_periodic_time_stamp = current; - _process_reference_time_s = process_time_s; - double collector_time_s = gc_thread_time_seconds(); - double elapsed_collector_time_s = collector_time_s - _collector_reference_time_s; - _collector_reference_time_s = collector_time_s; - double minimum_mutator_utilization = ((elapsed_process_time_s - elapsed_collector_time_s) / elapsed_process_time_s) * 100; - _mmu_average.add(minimum_mutator_utilization); - log_info(gc)("Average MMU = %.3f", _mmu_average.davg()); + double gc_time, mutator_time; + fetch_cpu_times(gc_time, mutator_time); + + double gc_delta = gc_time - _most_recent_periodic_gc_time; + _most_recent_periodic_gc_time = gc_time; + + double mutator_delta = mutator_time - _most_recent_periodic_mutator_time; + _most_recent_periodic_mutator_time = mutator_time; + + double mu = mutator_delta / (_active_processors * time_delta); + double gcu = gc_delta / (_active_processors * time_delta); + log_info(gc)("Periodic Sample: GCU = %.3f%%, MU = %.3f%% during most recent %.1fs", gcu * 100, mu * 100, time_delta); } void ShenandoahMmuTracker::initialize() { - _process_reference_time_s = process_time_seconds(); - _generational_reference_time_s = gc_thread_time_seconds(); - _collector_reference_time_s = _generational_reference_time_s; + // initialize static data + _active_processors = os::initial_active_processor_count(); + + double _most_recent_periodic_time_stamp = os::elapsedTime(); + fetch_cpu_times(_most_recent_periodic_gc_time, _most_recent_periodic_mutator_time); _mmu_periodic_task->enroll(); } @@ -160,12 +227,12 @@ ShenandoahGenerationSizer::ShenandoahGenerationSizer(ShenandoahMmuTracker* mmu_t size_t ShenandoahGenerationSizer::calculate_min_young_regions(size_t heap_region_count) { size_t min_young_regions = (heap_region_count * ShenandoahMinYoungPercentage) / 100; - return MAX2(uint(min_young_regions), 1U); + return MAX2(min_young_regions, (size_t) 1U); } size_t ShenandoahGenerationSizer::calculate_max_young_regions(size_t heap_region_count) { size_t max_young_regions = (heap_region_count * ShenandoahMaxYoungPercentage) / 100; - return MAX2(uint(max_young_regions), 1U); + return MAX2(max_young_regions, (size_t) 1U); } void ShenandoahGenerationSizer::recalculate_min_max_young_length(size_t heap_region_count) { @@ -202,110 +269,71 @@ void ShenandoahGenerationSizer::heap_size_changed(size_t heap_size) { recalculate_min_max_young_length(heap_size / ShenandoahHeapRegion::region_size_bytes()); } -bool ShenandoahGenerationSizer::adjust_generation_sizes() const { - shenandoah_assert_generational(); - if (!use_adaptive_sizing()) { - return false; - } - - if (_mmu_tracker->average() >= double(GCTimeRatio)) { - return false; - } - +// Returns true iff transfer is successful +bool ShenandoahGenerationSizer::transfer_to_old(size_t regions) const { ShenandoahHeap* heap = ShenandoahHeap::heap(); - ShenandoahOldGeneration *old = heap->old_generation(); - ShenandoahYoungGeneration *young = heap->young_generation(); - ShenandoahGeneration *global = heap->global_generation(); - double old_time_s = old->reset_collection_time(); - double young_time_s = young->reset_collection_time(); - double global_time_s = global->reset_collection_time(); - - const double transfer_threshold = 3.0; - double delta = young_time_s - old_time_s; - - log_info(gc)("Thread Usr+Sys YOUNG = %.3f, OLD = %.3f, GLOBAL = %.3f", young_time_s, old_time_s, global_time_s); + ShenandoahGeneration* old_gen = heap->old_generation(); + ShenandoahGeneration* young_gen = heap->young_generation(); + size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); + size_t bytes_to_transfer = regions * region_size_bytes; - if (abs(delta) <= transfer_threshold) { - log_info(gc, ergo)("Difference (%.3f) for thread utilization for each generation is under threshold (%.3f)", abs(delta), transfer_threshold); + if (young_gen->free_unaffiliated_regions() < regions) { + return false; + } else if (old_gen->max_capacity() + bytes_to_transfer > heap->max_size_for(old_gen)) { + return false; + } else if (young_gen->max_capacity() - bytes_to_transfer < heap->min_size_for(young_gen)) { return false; - } - - if (delta > 0) { - // young is busier than old, increase size of young to raise MMU - return transfer_capacity(old, young); } else { - // old is busier than young, increase size of old to raise MMU - return transfer_capacity(young, old); + young_gen->decrease_capacity(bytes_to_transfer); + old_gen->increase_capacity(bytes_to_transfer); + size_t new_size = old_gen->max_capacity(); + log_info(gc)("Transfer " SIZE_FORMAT " region(s) from %s to %s, yielding increased size: " SIZE_FORMAT "%s", + regions, young_gen->name(), old_gen->name(), + byte_size_in_proper_unit(new_size), proper_unit_for_byte_size(new_size)); + return true; } } -bool ShenandoahGenerationSizer::transfer_capacity(ShenandoahGeneration* target) const { - ShenandoahHeapLocker locker(ShenandoahHeap::heap()->lock()); - if (target->is_young()) { - return transfer_capacity(ShenandoahHeap::heap()->old_generation(), target); - } else { - assert(target->is_old(), "Expected old generation, if not young."); - return transfer_capacity(ShenandoahHeap::heap()->young_generation(), target); - } +// This is used when promoting humongous or highly utilized regular regions in place. It is not required in this situation +// that the transferred regions be unaffiliated. +void ShenandoahGenerationSizer::force_transfer_to_old(size_t regions) const { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + ShenandoahGeneration* old_gen = heap->old_generation(); + ShenandoahGeneration* young_gen = heap->young_generation(); + size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); + size_t bytes_to_transfer = regions * region_size_bytes; + + young_gen->decrease_capacity(bytes_to_transfer); + old_gen->increase_capacity(bytes_to_transfer); + size_t new_size = old_gen->max_capacity(); + log_info(gc)("Forcing transfer of " SIZE_FORMAT " region(s) from %s to %s, yielding increased size: " SIZE_FORMAT "%s", + regions, young_gen->name(), old_gen->name(), + byte_size_in_proper_unit(new_size), proper_unit_for_byte_size(new_size)); } -bool ShenandoahGenerationSizer::transfer_capacity(ShenandoahGeneration* from, ShenandoahGeneration* to) const { - shenandoah_assert_heaplocked_or_safepoint(); - size_t available_regions = from->free_unaffiliated_regions(); - if (available_regions <= 0) { - log_info(gc)("%s has no regions available for transfer to %s", from->name(), to->name()); - return false; - } - - size_t regions_to_transfer = MAX2(1u, uint(double(available_regions) * _resize_increment)); - if (from->is_young()) { - regions_to_transfer = adjust_transfer_from_young(from, regions_to_transfer); - } else { - regions_to_transfer = adjust_transfer_to_young(to, regions_to_transfer); - } +bool ShenandoahGenerationSizer::transfer_to_young(size_t regions) const { + ShenandoahHeap* heap = ShenandoahHeap::heap(); + ShenandoahGeneration* old_gen = heap->old_generation(); + ShenandoahGeneration* young_gen = heap->young_generation(); + size_t region_size_bytes = ShenandoahHeapRegion::region_size_bytes(); + size_t bytes_to_transfer = regions * region_size_bytes; - if (regions_to_transfer == 0) { - log_info(gc)("No capacity available to transfer from: %s (" SIZE_FORMAT "%s) to: %s (" SIZE_FORMAT "%s)", - from->name(), byte_size_in_proper_unit(from->max_capacity()), proper_unit_for_byte_size(from->max_capacity()), - to->name(), byte_size_in_proper_unit(to->max_capacity()), proper_unit_for_byte_size(to->max_capacity())); + if (old_gen->free_unaffiliated_regions() < regions) { return false; + } else if (young_gen->max_capacity() + bytes_to_transfer > heap->max_size_for(young_gen)) { + return false; + } else if (old_gen->max_capacity() - bytes_to_transfer < heap->min_size_for(old_gen)) { + return false; + } else { + old_gen->decrease_capacity(bytes_to_transfer); + young_gen->increase_capacity(bytes_to_transfer); + size_t new_size = young_gen->max_capacity(); + log_info(gc)("Transfer " SIZE_FORMAT " region(s) from %s to %s, yielding increased size: " SIZE_FORMAT "%s", + regions, old_gen->name(), young_gen->name(), + byte_size_in_proper_unit(new_size), proper_unit_for_byte_size(new_size)); + return true; } - - log_info(gc)("Transfer " SIZE_FORMAT " region(s) from %s to %s", regions_to_transfer, from->name(), to->name()); - from->decrease_capacity(regions_to_transfer * ShenandoahHeapRegion::region_size_bytes()); - to->increase_capacity(regions_to_transfer * ShenandoahHeapRegion::region_size_bytes()); - return true; -} - -size_t ShenandoahGenerationSizer::adjust_transfer_from_young(ShenandoahGeneration* from, size_t regions_to_transfer) const { - assert(from->is_young(), "Expect to transfer from young"); - size_t young_capacity_regions = from->max_capacity() / ShenandoahHeapRegion::region_size_bytes(); - size_t new_young_regions = young_capacity_regions - regions_to_transfer; - size_t minimum_young_regions = min_young_regions(); - // Check that we are not going to violate the minimum size constraint. - if (new_young_regions < minimum_young_regions) { - assert(minimum_young_regions <= young_capacity_regions, "Young is under minimum capacity."); - // If the transfer violates the minimum size and there is still some capacity to transfer, - // adjust the transfer to take the size to the minimum. Note that this may be zero. - regions_to_transfer = young_capacity_regions - minimum_young_regions; - } - return regions_to_transfer; -} - -size_t ShenandoahGenerationSizer::adjust_transfer_to_young(ShenandoahGeneration* to, size_t regions_to_transfer) const { - assert(to->is_young(), "Can only transfer between young and old."); - size_t young_capacity_regions = to->max_capacity() / ShenandoahHeapRegion::region_size_bytes(); - size_t new_young_regions = young_capacity_regions + regions_to_transfer; - size_t maximum_young_regions = max_young_regions(); - // Check that we are not going to violate the maximum size constraint. - if (new_young_regions > maximum_young_regions) { - assert(maximum_young_regions >= young_capacity_regions, "Young is over maximum capacity"); - // If the transfer violates the maximum size and there is still some capacity to transfer, - // adjust the transfer to take the size to the maximum. Note that this may be zero. - regions_to_transfer = maximum_young_regions - young_capacity_regions; - } - return regions_to_transfer; } size_t ShenandoahGenerationSizer::min_young_size() const { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.hpp b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.hpp index 9d3c230a6cc57..6ab7d18024583 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahMmuTracker.hpp @@ -50,16 +50,31 @@ class ShenandoahMmuTask; * MMU. */ class ShenandoahMmuTracker { +private: + // These variables hold recent snapshots of cumulative quantities that are used for calculating + // CPU time consumed by GC and mutator threads during each GC cycle. + double _most_recent_timestamp; + double _most_recent_gc_time; + double _most_recent_gcu; + double _most_recent_mutator_time; + double _most_recent_mu; + + // These variables hold recent snapshots of cumulative quantities that are used for reporting + // periodic consumption of CPU time by GC and mutator threads. + double _most_recent_periodic_time_stamp; + double _most_recent_periodic_gc_time; + double _most_recent_periodic_mutator_time; - double _generational_reference_time_s; - double _process_reference_time_s; - double _collector_reference_time_s; + size_t _most_recent_gcid; + uint _active_processors; + + bool _most_recent_is_full; ShenandoahMmuTask* _mmu_periodic_task; TruncatedSeq _mmu_average; - static double gc_thread_time_seconds(); - static double process_time_seconds(); + void update_utilization(ShenandoahGeneration* generation, size_t gcid, const char* msg); + static void fetch_cpu_times(double &gc_time, double &mutator_time); public: explicit ShenandoahMmuTracker(); @@ -68,22 +83,24 @@ class ShenandoahMmuTracker { // This enrolls the periodic task after everything is initialized. void initialize(); - // This is called at the start and end of a GC cycle. The GC thread times - // will be accumulated in this generation. Note that the bootstrap cycle - // for an old collection should be counted against the old generation. - // When the collector is idle, it still runs a regulator and a control. - // The times for these threads are attributed to the global generation. - void record(ShenandoahGeneration* generation); + // At completion of each GC cycle (not including interrupted cycles), we invoke one of the following to record the + // GC utilization during this cycle. Incremental efforts spent in an interrupted GC cycle will be accumulated into + // the CPU time reports for the subsequent completed [degenerated or full] GC cycle. + // + // We may redundantly record degen and full in the case that a degen upgrades to full. When this happens, we will invoke + // both record_full() and record_degenerated() with the same value of gcid. record_full() is called first and the log + // reports such a cycle as a FULL cycle. + void record_young(ShenandoahGeneration* generation, size_t gcid); + void record_bootstrap(ShenandoahGeneration* generation, size_t gcid, bool has_old_candidates); + void record_old_marking_increment(ShenandoahGeneration* generation, size_t gcid, bool old_marking_done, bool has_old_candidates); + void record_mixed(ShenandoahGeneration* generation, size_t gcid, bool is_mixed_done); + void record_full(ShenandoahGeneration* generation, size_t gcid); + void record_degenerated(ShenandoahGeneration* generation, size_t gcid, bool is_old_boostrap, bool is_mixed_done); // This is called by the periodic task timer. The interval is defined by // GCPauseIntervalMillis and defaults to 5 seconds. This method computes // the MMU over the elapsed interval and records it in a running average. - // This method also logs the average MMU. void report(); - - double average() { - return _mmu_average.davg(); - } }; class ShenandoahGenerationSizer { @@ -114,14 +131,6 @@ class ShenandoahGenerationSizer { // given the number of heap regions depending on the kind of sizing algorithm. void recalculate_min_max_young_length(size_t heap_region_count); - // These two methods are responsible for enforcing the minimum and maximum - // constraints for the size of the generations. - size_t adjust_transfer_from_young(ShenandoahGeneration* from, size_t regions_to_transfer) const; - size_t adjust_transfer_to_young(ShenandoahGeneration* to, size_t regions_to_transfer) const; - - // This will attempt to transfer capacity from one generation to the other. It - // returns true if a transfer is made, false otherwise. - bool transfer_capacity(ShenandoahGeneration* from, ShenandoahGeneration* to) const; public: explicit ShenandoahGenerationSizer(ShenandoahMmuTracker* mmu_tracker); @@ -145,19 +154,11 @@ class ShenandoahGenerationSizer { return _use_adaptive_sizing; } - // This is invoked at the end of a collection. This happens on a safepoint - // to avoid any races with allocators (and to avoid interfering with - // allocators by taking the heap lock). The amount of capacity to move - // from one generation to another is controlled by YoungGenerationSizeIncrement - // and defaults to 20% of the available capacity of the donor generation. - // The minimum and maximum sizes of the young generation are controlled by - // ShenandoahMinYoungPercentage and ShenandoahMaxYoungPercentage, respectively. - // The method returns true when an adjustment is made, false otherwise. - bool adjust_generation_sizes() const; - - // This may be invoked by a heuristic (from regulator thread) before it - // decides to run a collection. - bool transfer_capacity(ShenandoahGeneration* target) const; + bool transfer_to_young(size_t regions) const; + bool transfer_to_old(size_t regions) const; + + // force transfer is used when we promote humongous objects. May violate min/max limits on generation sizes + void force_transfer_to_old(size_t regions) const; }; #endif //SHARE_GC_SHENANDOAH_SHENANDOAHMMUTRACKER_HPP diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp index 94e29209bac5c..da56f2cb6831f 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGC.cpp @@ -30,6 +30,7 @@ #include "gc/shenandoah/shenandoahOldGC.hpp" #include "gc/shenandoah/shenandoahOopClosures.inline.hpp" #include "gc/shenandoah/shenandoahGeneration.hpp" +#include "gc/shenandoah/shenandoahOldGeneration.hpp" #include "gc/shenandoah/shenandoahYoungGeneration.hpp" #include "prims/jvmtiTagMap.hpp" #include "utilities/events.hpp" @@ -146,5 +147,48 @@ bool ShenandoahOldGC::collect(GCCause::Cause cause) { // collection. vmop_entry_final_roots(); + // We do not rebuild_free following increments of old marking because memory has not been reclaimed.. However, we may + // need to transfer memory to OLD in order to efficiently support the mixed evacuations that might immediately follow. + size_t allocation_runway = heap->young_generation()->heuristics()->bytes_of_allocation_runway_before_gc_trigger(0); + heap->adjust_generation_sizes_for_next_cycle(allocation_runway, 0, 0); + + bool success; + size_t region_xfer; + const char* region_destination; + ShenandoahYoungGeneration* young_gen = heap->young_generation(); + ShenandoahGeneration* old_gen = heap->old_generation(); + { + ShenandoahHeapLocker locker(heap->lock()); + + size_t old_region_surplus = heap->get_old_region_surplus(); + size_t old_region_deficit = heap->get_old_region_deficit(); + if (old_region_surplus) { + success = heap->generation_sizer()->transfer_to_young(old_region_surplus); + region_destination = "young"; + region_xfer = old_region_surplus; + } else if (old_region_deficit) { + success = heap->generation_sizer()->transfer_to_old(old_region_deficit); + region_destination = "old"; + region_xfer = old_region_deficit; + if (!success) { + ((ShenandoahOldHeuristics *) old_gen->heuristics())->trigger_cannot_expand(); + } + } else { + region_destination = "none"; + region_xfer = 0; + success = true; + } + heap->set_old_region_surplus(0); + heap->set_old_region_deficit(0); + } + + // Report outside the heap lock + size_t young_available = young_gen->available(); + size_t old_available = old_gen->available(); + log_info(gc, ergo)("After old marking finished, %s " SIZE_FORMAT " regions to %s to prepare for next gc, old available: " + SIZE_FORMAT "%s, young_available: " SIZE_FORMAT "%s", + success? "successfully transferred": "failed to transfer", region_xfer, region_destination, + byte_size_in_proper_unit(old_available), proper_unit_for_byte_size(old_available), + byte_size_in_proper_unit(young_available), proper_unit_for_byte_size(young_available)); return true; } diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp index 331499816f815..f8aaf325cbdb7 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.cpp @@ -174,12 +174,30 @@ class ShenandoahConcurrentCoalesceAndFillTask : public WorkerTask { ShenandoahOldGeneration::ShenandoahOldGeneration(uint max_queues, size_t max_capacity, size_t soft_max_capacity) : ShenandoahGeneration(OLD, max_queues, max_capacity, soft_max_capacity), _coalesce_and_fill_region_array(NEW_C_HEAP_ARRAY(ShenandoahHeapRegion*, ShenandoahHeap::heap()->num_regions(), mtGC)), - _state(IDLE) + _state(IDLE), + _growth_before_compaction(INITIAL_GROWTH_BEFORE_COMPACTION) { + _live_bytes_after_last_mark = ShenandoahHeap::heap()->capacity() * INITIAL_LIVE_FRACTION / FRACTIONAL_DENOMINATOR; // Always clear references for old generation ref_processor()->set_soft_reference_policy(true); } +size_t ShenandoahOldGeneration::get_live_bytes_after_last_mark() const { + return _live_bytes_after_last_mark; +} + +void ShenandoahOldGeneration::set_live_bytes_after_last_mark(size_t bytes) { + _live_bytes_after_last_mark = bytes; + if (_growth_before_compaction > MINIMUM_GROWTH_BEFORE_COMPACTION) { + _growth_before_compaction /= 2; + } +} + +size_t ShenandoahOldGeneration::usage_trigger_threshold() const { + size_t result = _live_bytes_after_last_mark + (_live_bytes_after_last_mark * _growth_before_compaction) / FRACTIONAL_DENOMINATOR; + return result; +} + bool ShenandoahOldGeneration::contains(ShenandoahHeapRegion* region) const { // TODO: Should this be region->is_old() instead? return !region->is_young(); @@ -255,6 +273,7 @@ bool ShenandoahOldGeneration::coalesce_and_fill() { uint nworkers = workers->active_workers(); log_debug(gc)("Starting (or resuming) coalesce-and-fill of old heap regions"); + // This code will see the same set of regions to fill on each resumption as it did // on the initial run. That's okay because each region keeps track of its own coalesce // and fill state. Regions that were filled on a prior attempt will not try to fill again. @@ -322,7 +341,11 @@ void ShenandoahOldGeneration::prepare_regions_and_collection_set(bool concurrent ShenandoahPhaseTimings::final_rebuild_freeset : ShenandoahPhaseTimings::degen_gc_final_rebuild_freeset); ShenandoahHeapLocker locker(heap->lock()); - heap->free_set()->rebuild(); + size_t cset_young_regions, cset_old_regions; + heap->free_set()->prepare_to_rebuild(cset_young_regions, cset_old_regions); + // This is just old-gen completion. No future budgeting required here. The only reason to rebuild the freeset here + // is in case there was any immediate old garbage identified. + heap->free_set()->rebuild(cset_young_regions, cset_old_regions); } } @@ -405,7 +428,8 @@ void ShenandoahOldGeneration::validate_transition(State new_state) { case IDLE: // GC cancellation can send us back to IDLE from any state. assert(!heap->is_concurrent_old_mark_in_progress(), "Cannot become idle during old mark."); - assert(_old_heuristics->unprocessed_old_collection_candidates() == 0, "Cannot become idle with collection candidates"); + assert(!heap->mode()->is_generational() || + (_old_heuristics->unprocessed_old_collection_candidates() == 0), "Cannot become idle with collection candidates"); assert(!heap->is_prepare_for_old_mark_in_progress(), "Cannot become idle while making old generation parseable."); assert(heap->young_generation()->old_gen_task_queues() == nullptr, "Cannot become idle when setup for bootstrapping."); break; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp index 0e3fb429c2634..1ac7e22c41a51 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahOldGeneration.hpp @@ -90,19 +90,45 @@ class ShenandoahOldGeneration : public ShenandoahGeneration { private: State _state; + static const size_t FRACTIONAL_DENOMINATOR = 64536; + + // During initialization of the JVM, we search for the correct old-gen size by initally performing old-gen + // collection when old-gen usage is 50% more (INITIAL_GROWTH_BEFORE_COMPACTION) than the initial old-gen size + // estimate (3.125% of heap). The next old-gen trigger occurs when old-gen grows 25% larger than its live + // memory at the end of the first old-gen collection. Then we trigger again when old-gen growns 12.5% + // more than its live memory at the end of the previous old-gen collection. Thereafter, we trigger each time + // old-gen grows more than 12.5% following the end of its previous old-gen collection. + static const size_t INITIAL_GROWTH_BEFORE_COMPACTION = FRACTIONAL_DENOMINATOR / 2; // 50.0% + static const size_t MINIMUM_GROWTH_BEFORE_COMPACTION = FRACTIONAL_DENOMINATOR / 8; // 12.5% + + // INITIAL_LIVE_FRACTION represents the initial guess of how large old-gen should be. We estimate that old-gen + // needs to consume 3.125% of the total heap size. And we "pretend" that we start out with this amount of live + // old-gen memory. The first old-collection trigger will occur when old-gen occupies 50% more than this initial + // approximation of the old-gen memory requirement, in other words when old-gen usage is 150% of 3.125%, which + // is 4.6875% of the total heap size. + static const uint16_t INITIAL_LIVE_FRACTION = FRACTIONAL_DENOMINATOR / 32; // 3.125% + size_t _live_bytes_after_last_mark; + size_t _growth_before_compaction; // How much growth in usage before we trigger old collection, per 65_536 + + void validate_transition(State new_state) NOT_DEBUG_RETURN; + public: State state() const { return _state; } + void transition_to(State new_state); + + size_t get_live_bytes_after_last_mark() const; + void set_live_bytes_after_last_mark(size_t new_live); + + size_t usage_trigger_threshold() const; + bool can_start_gc() { return _state == IDLE || _state == WAITING_FOR_FILL; } static const char* state_name(State state); - - void transition_to(State new_state); - void validate_transition(State new_state) NOT_DEBUG_RETURN; }; diff --git a/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.cpp b/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.cpp index 7664b75d1d649..7d21fb27972c8 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahRegulatorThread.cpp @@ -140,7 +140,8 @@ void ShenandoahRegulatorThread::regulator_sleep() { } bool ShenandoahRegulatorThread::start_old_cycle() { - return _old_heuristics->should_start_gc() && _control_thread->request_concurrent_gc(OLD); + return !ShenandoahHeap::heap()->doing_mixed_evacuations() && !ShenandoahHeap::heap()->collection_set()->has_old_regions() && + _old_heuristics->should_start_gc() && _control_thread->request_concurrent_gc(OLD); } bool ShenandoahRegulatorThread::start_young_cycle() { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp index dae7ab21b2d28..61ea96999d024 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.cpp @@ -403,24 +403,32 @@ class ShenandoahGenerationStatsClosure : public ShenandoahHeapRegionClosure { byte_size_in_proper_unit(stats.used()), proper_unit_for_byte_size(stats.used())); } - static void validate_usage(const char* label, ShenandoahGeneration* generation, ShenandoahCalculateRegionStatsClosure& stats) { + static void validate_usage(const bool adjust_for_padding, + const char* label, ShenandoahGeneration* generation, ShenandoahCalculateRegionStatsClosure& stats) { + ShenandoahHeap* heap = ShenandoahHeap::heap(); size_t generation_used = generation->used(); + size_t generation_used_regions = generation->used_regions(); + if (adjust_for_padding && (generation->is_young() || generation->is_global())) { + size_t pad = ShenandoahHeap::heap()->get_pad_for_promote_in_place(); + generation_used += pad; + } + guarantee(stats.used() == generation_used, "%s: generation (%s) used size must be consistent: generation-used: " SIZE_FORMAT "%s, regions-used: " SIZE_FORMAT "%s", label, generation->name(), byte_size_in_proper_unit(generation_used), proper_unit_for_byte_size(generation_used), byte_size_in_proper_unit(stats.used()), proper_unit_for_byte_size(stats.used())); - guarantee(stats.regions() == generation->used_regions(), + guarantee(stats.regions() == generation_used_regions, "%s: generation (%s) used regions (" SIZE_FORMAT ") must equal regions that are in use (" SIZE_FORMAT ")", label, generation->name(), generation->used_regions(), stats.regions()); -// This check is disabled because of known issues with this feature. We expect this code to be updated by 05/2023. -// size_t capacity = generation->adjusted_capacity(); -// guarantee(stats.span() <= capacity, -// "%s: generation (%s) size spanned by regions (" SIZE_FORMAT ") must not exceed current capacity (" SIZE_FORMAT "%s)", -// label, generation->name(), stats.regions(), -// byte_size_in_proper_unit(capacity), proper_unit_for_byte_size(capacity)); + size_t generation_capacity = generation->max_capacity(); + size_t humongous_regions_promoted = 0; + guarantee(stats.span() <= generation_capacity, + "%s: generation (%s) size spanned by regions (" SIZE_FORMAT ") must not exceed current capacity (" SIZE_FORMAT "%s)", + label, generation->name(), stats.regions(), + byte_size_in_proper_unit(generation_capacity), proper_unit_for_byte_size(generation_capacity)); size_t humongous_waste = generation->get_humongous_waste(); guarantee(stats.waste() == humongous_waste, @@ -743,6 +751,7 @@ void ShenandoahVerifier::verify_at_safepoint(const char* label, VerifyForwarded forwarded, VerifyMarked marked, VerifyCollectionSet cset, VerifyLiveness liveness, VerifyRegions regions, + VerifySize sizeness, VerifyGCState gcstate) { guarantee(ShenandoahSafepoint::is_at_shenandoah_safepoint(), "only when nothing else happens"); guarantee(ShenandoahVerify, "only when enabled, and bitmap is initialized in ShenandoahHeap::initialize"); @@ -795,6 +804,7 @@ void ShenandoahVerifier::verify_at_safepoint(const char* label, if (enabled) { char actual = _heap->gc_state(); + // Old generation marking is allowed in all states. if (!VerifyThreadGCState::verify_gc_state(actual, expected)) { fatal("%s: Global gc-state: expected %d, actual %d", label, expected, actual); } @@ -813,13 +823,20 @@ void ShenandoahVerifier::verify_at_safepoint(const char* label, ShenandoahCalculateRegionStatsClosure cl; _heap->heap_region_iterate(&cl); - size_t heap_used = _heap->used(); - guarantee(cl.used() == heap_used, - "%s: heap used size must be consistent: heap-used = " SIZE_FORMAT "%s, regions-used = " SIZE_FORMAT "%s", - label, - byte_size_in_proper_unit(heap_used), proper_unit_for_byte_size(heap_used), - byte_size_in_proper_unit(cl.used()), proper_unit_for_byte_size(cl.used())); - + size_t heap_used; + if (_heap->mode()->is_generational() && (sizeness == _verify_size_adjusted_for_padding)) { + // Prior to evacuation, regular regions that are to be evacuated in place are padded to prevent further allocations + heap_used = _heap->used() + _heap->get_pad_for_promote_in_place(); + } else if (sizeness != _verify_size_disable) { + heap_used = _heap->used(); + } + if (sizeness != _verify_size_disable) { + guarantee(cl.used() == heap_used, + "%s: heap used size must be consistent: heap-used = " SIZE_FORMAT "%s, regions-used = " SIZE_FORMAT "%s", + label, + byte_size_in_proper_unit(heap_used), proper_unit_for_byte_size(heap_used), + byte_size_in_proper_unit(cl.used()), proper_unit_for_byte_size(cl.used())); + } size_t heap_committed = _heap->committed(); guarantee(cl.committed() == heap_committed, "%s: heap committed size must be consistent: heap-committed = " SIZE_FORMAT "%s, regions-committed = " SIZE_FORMAT "%s", @@ -868,10 +885,16 @@ void ShenandoahVerifier::verify_at_safepoint(const char* label, ShenandoahGenerationStatsClosure::log_usage(_heap->young_generation(), cl.young); ShenandoahGenerationStatsClosure::log_usage(_heap->global_generation(), cl.global); } - - ShenandoahGenerationStatsClosure::validate_usage(label, _heap->old_generation(), cl.old); - ShenandoahGenerationStatsClosure::validate_usage(label, _heap->young_generation(), cl.young); - ShenandoahGenerationStatsClosure::validate_usage(label, _heap->global_generation(), cl.global); + if (sizeness == _verify_size_adjusted_for_padding) { + ShenandoahGenerationStatsClosure::validate_usage(false, label, _heap->old_generation(), cl.old); + ShenandoahGenerationStatsClosure::validate_usage(true, label, _heap->young_generation(), cl.young); + ShenandoahGenerationStatsClosure::validate_usage(true, label, _heap->global_generation(), cl.global); + } else if (sizeness == _verify_size_exact) { + ShenandoahGenerationStatsClosure::validate_usage(false, label, _heap->old_generation(), cl.old); + ShenandoahGenerationStatsClosure::validate_usage(false, label, _heap->young_generation(), cl.young); + ShenandoahGenerationStatsClosure::validate_usage(false, label, _heap->global_generation(), cl.global); + } + // else: sizeness must equal _verify_size_disable } log_debug(gc)("Safepoint verification finished remembered set verification"); @@ -983,19 +1006,22 @@ void ShenandoahVerifier::verify_generic(VerifyOption vo) { _verify_cset_disable, // cset may be inconsistent _verify_liveness_disable, // no reliable liveness data _verify_regions_disable, // no reliable region data + _verify_size_exact, // expect generation and heap sizes to match exactly _verify_gcstate_disable // no data about gcstate ); } void ShenandoahVerifier::verify_before_concmark() { verify_at_safepoint( - "Before Mark", - _verify_remembered_before_marking, // verify read-only remembered set from bottom() to top() + "Before Mark", + _verify_remembered_before_marking, + // verify read-only remembered set from bottom() to top() _verify_forwarded_none, // UR should have fixed up _verify_marked_disable, // do not verify marked: lots ot time wasted checking dead allocations _verify_cset_none, // UR should have fixed this _verify_liveness_disable, // no reliable liveness data _verify_regions_notrash, // no trash regions + _verify_size_exact, // expect generation and heap sizes to match exactly _verify_gcstate_stable // there are no forwarded objects ); } @@ -1005,10 +1031,12 @@ void ShenandoahVerifier::verify_after_concmark() { "After Mark", _verify_remembered_disable, // do not verify remembered set _verify_forwarded_none, // no forwarded references - _verify_marked_complete_except_references, // bitmaps as precise as we can get, except dangling j.l.r.Refs + _verify_marked_complete_except_references, + // bitmaps as precise as we can get, except dangling j.l.r.Refs _verify_cset_none, // no references to cset anymore _verify_liveness_complete, // liveness data must be complete here _verify_regions_disable, // trash regions not yet recycled + _verify_size_exact, // expect generation and heap sizes to match exactly _verify_gcstate_stable_weakroots // heap is still stable, weakroots are in progress ); } @@ -1022,6 +1050,8 @@ void ShenandoahVerifier::verify_before_evacuation() { _verify_cset_disable, // non-forwarded references to cset expected _verify_liveness_complete, // liveness data must be complete here _verify_regions_disable, // trash regions not yet recycled + _verify_size_adjusted_for_padding, // expect generation and heap sizes to match after adjustments + // for promote in place padding _verify_gcstate_stable_weakroots // heap is still stable, weakroots are in progress ); } @@ -1035,6 +1065,7 @@ void ShenandoahVerifier::verify_during_evacuation() { _verify_cset_disable, // some cset references are not forwarded yet _verify_liveness_disable, // liveness data might be already stale after pre-evacs _verify_regions_disable, // trash regions not yet recycled + _verify_size_disable, // we don't know how much of promote-in-place work has been completed _verify_gcstate_evacuation // evacuation is in progress ); } @@ -1048,6 +1079,7 @@ void ShenandoahVerifier::verify_after_evacuation() { _verify_cset_forwarded, // all cset refs are fully forwarded _verify_liveness_disable, // no reliable liveness data anymore _verify_regions_notrash, // trash regions have been recycled already + _verify_size_exact, // expect generation and heap sizes to match exactly _verify_gcstate_forwarded // evacuation produced some forwarded objects ); } @@ -1056,15 +1088,17 @@ void ShenandoahVerifier::verify_before_updaterefs() { verify_at_safepoint( "Before Updating References", _verify_remembered_before_updating_references, // verify read-write remembered set - _verify_forwarded_allow, // forwarded references allowed - _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well - _verify_cset_forwarded, // all cset refs are fully forwarded - _verify_liveness_disable, // no reliable liveness data anymore - _verify_regions_notrash, // trash regions have been recycled already - _verify_gcstate_updating // evacuation should have produced some forwarded objects + _verify_forwarded_allow, // forwarded references allowed + _verify_marked_complete, // bitmaps might be stale, but alloc-after-mark should be well + _verify_cset_forwarded, // all cset refs are fully forwarded + _verify_liveness_disable, // no reliable liveness data anymore + _verify_regions_notrash, // trash regions have been recycled already + _verify_size_exact, // expect generation and heap sizes to match exactly + _verify_gcstate_updating // evacuation should have produced some forwarded objects ); } +// We have not yet cleanup (reclaimed) the collection set void ShenandoahVerifier::verify_after_updaterefs() { verify_at_safepoint( "After Updating References", @@ -1074,6 +1108,7 @@ void ShenandoahVerifier::verify_after_updaterefs() { _verify_cset_none, // no cset references, all updated _verify_liveness_disable, // no reliable liveness data anymore _verify_regions_nocset, // no cset regions, trash regions have appeared + _verify_size_exact, // expect generation and heap sizes to match exactly _verify_gcstate_stable // update refs had cleaned up forwarded objects ); } @@ -1087,6 +1122,7 @@ void ShenandoahVerifier::verify_after_degenerated() { _verify_cset_none, // no cset references _verify_liveness_disable, // no reliable liveness data anymore _verify_regions_notrash_nocset, // no trash, no cset + _verify_size_exact, // expect generation and heap sizes to match exactly _verify_gcstate_stable // degenerated refs had cleaned up forwarded objects ); } @@ -1100,6 +1136,7 @@ void ShenandoahVerifier::verify_before_fullgc() { _verify_cset_disable, // cset might be foobared _verify_liveness_disable, // no reliable liveness data anymore _verify_regions_disable, // no reliable region data here + _verify_size_disable, // if we degenerate during evacuation, usage not valid: padding and deferred accounting _verify_gcstate_disable // no reliable gcstate data ); } @@ -1113,6 +1150,7 @@ void ShenandoahVerifier::verify_after_fullgc() { _verify_cset_none, // no cset references _verify_liveness_disable, // no reliable liveness data anymore _verify_regions_notrash_nocset, // no trash, no cset + _verify_size_exact, // expect generation and heap sizes to match exactly _verify_gcstate_stable // full gc cleaned up everything ); } @@ -1314,7 +1352,7 @@ void ShenandoahVerifier::verify_rem_set_before_mark() { } // else, this humongous object is not marked so no need to verify its internal pointers if (!scanner->verify_registration(obj_addr, ctx)) { - ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, obj_addr, nullptr, + ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, nullptr, nullptr, "Verify init-mark remembered set violation", "object not properly registered", __FILE__, __LINE__); } } else if (!r->is_humongous()) { @@ -1330,7 +1368,7 @@ void ShenandoahVerifier::verify_rem_set_before_mark() { } // else, object's start is marked dirty and obj is not an objArray, so any interesting pointers are covered if (!scanner->verify_registration(obj_addr, ctx)) { - ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, obj_addr, nullptr, + ShenandoahAsserts::print_failure(ShenandoahAsserts::_safe_all, obj, nullptr, nullptr, "Verify init-mark remembered set violation", "object not properly registered", __FILE__, __LINE__); } obj_addr += obj->size(); diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp index 284746be54108..67b35644bf532 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVerifier.hpp @@ -141,6 +141,17 @@ class ShenandoahVerifier : public CHeapObj { _verify_regions_notrash_nocset } VerifyRegions; + typedef enum { + // Disable size verification + _verify_size_disable, + + // Enforce exact consistency + _verify_size_exact, + + // Expect promote-in-place adjustments: padding inserted to temporarily prevent further allocation in regular regions + _verify_size_adjusted_for_padding + } VerifySize; + typedef enum { // Disable gc-state verification _verify_gcstate_disable, @@ -189,6 +200,7 @@ class ShenandoahVerifier : public CHeapObj { VerifyCollectionSet cset, VerifyLiveness liveness, VerifyRegions regions, + VerifySize sizeness, VerifyGCState gcstate); public: diff --git a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp index a7f3311b603b3..4d5d029f4f754 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoah_globals.hpp @@ -97,7 +97,7 @@ "collector accepts. In percents of heap region size.") \ range(0,100) \ \ - product(uintx, ShenandoahOldGarbageThreshold, 10, EXPERIMENTAL, \ + product(uintx, ShenandoahOldGarbageThreshold, 15, EXPERIMENTAL, \ "How much garbage an old region has to contain before it would " \ "be taken for collection.") \ range(0,100) \ @@ -128,13 +128,6 @@ "size.") \ range(0,100) \ \ - product(uintx, ShenandoahOldMinFreeThreshold, 5, EXPERIMENTAL, \ - "Percentage of free old generation heap memory below which most " \ - "heuristics trigger collection independent of other triggers. " \ - "Provides a safety margin for many heuristics. In percents of " \ - "(soft) max heap size.") \ - range(0,100) \ - \ product(uintx, ShenandoahAllocationThreshold, 0, EXPERIMENTAL, \ "How many new allocations should happen since the last GC cycle " \ "before some heuristics trigger the collection. In percents of " \ @@ -210,7 +203,7 @@ "Heuristics may trigger collections more frequently. Time is in " \ "milliseconds. Setting this to 0 disables the feature.") \ \ - product(uintx, ShenandoahGuaranteedYoungGCInterval, 5*60*1000, EXPERIMENTAL, \ + product(uintx, ShenandoahGuaranteedYoungGCInterval, 5*60*1000, EXPERIMENTAL, \ "Run a collection of the young generation at least this often. " \ "Heuristics may trigger collections more frequently. Time is in " \ "milliseconds. Setting this to 0 disables the feature.") \ @@ -300,17 +293,20 @@ "failures, which will trigger stop-the-world Full GC passes.") \ range(1.0,100.0) \ \ - product(double, ShenandoahGenerationalEvacWaste, 2.0, EXPERIMENTAL, \ - "For generational mode, how much waste evacuations produce " \ - "within the reserved space. Larger values make evacuations " \ - "more resilient against evacuation conflicts, at expense of " \ - "evacuating less on each GC cycle. Smaller values increase " \ - "the risk of evacuation failures, which will trigger " \ - "stop-the-world Full GC passes. The default value for " \ - "generational mode is 2.0. The reason for the higher default " \ - "value in generational mode is because generational mode " \ - "enforces the evacuation budget, triggering degenerated GC " \ - "which upgrades to full GC whenever the budget is exceeded.") \ + product(double, ShenandoahOldEvacWaste, 1.4, EXPERIMENTAL, \ + "How much waste evacuations produce within the reserved space. " \ + "Larger values make evacuations more resilient against " \ + "evacuation conflicts, at expense of evacuating less on each " \ + "GC cycle. Smaller values increase the risk of evacuation " \ + "failures, which will trigger stop-the-world Full GC passes.") \ + range(1.0,100.0) \ + \ + product(double, ShenandoahPromoEvacWaste, 1.2, EXPERIMENTAL, \ + "How much waste promotions produce within the reserved space. " \ + "Larger values make evacuations more resilient against " \ + "evacuation conflicts, at expense of promoting less on each " \ + "GC cycle. Smaller values increase the risk of evacuation " \ + "failures, which will trigger stop-the-world Full GC passes.") \ range(1.0,100.0) \ \ product(uintx, ShenandoahMaxEvacLABRatio, 0, EXPERIMENTAL, \ @@ -339,26 +335,16 @@ "reserve/waste is incorrect, at the risk that application " \ "runs out of memory too early.") \ \ - product(uintx, ShenandoahOldEvacReserve, 2, EXPERIMENTAL, \ - "How much of old-generation heap to reserve for old-generation " \ - "evacuations. Larger values allow GC to evacuate more live " \ - "old-generation objects on every cycle, while potentially " \ - "creating greater impact on the cadence at which the young- " \ - "generation allocation pool is replenished. During mixed " \ - "evacuations, the bound on amount of old-generation heap " \ - "regions included in the collecdtion set is the smaller " \ - "of the quantities specified by this parameter and the " \ - "size of ShenandoahEvacReserve as adjusted by the value of " \ - "ShenandoahOldEvacRatioPercent. In percents of total " \ - "old-generation heap size.") \ - range(1,100) \ - \ - product(uintx, ShenandoahOldEvacRatioPercent, 12, EXPERIMENTAL, \ + product(uintx, ShenandoahOldEvacRatioPercent, 75, EXPERIMENTAL, \ "The maximum proportion of evacuation from old-gen memory, as " \ - "a percent ratio. The default value 12 denotes that no more " \ - "than one eighth (12%) of the collection set evacuation " \ - "workload may be comprised of old-gen heap regions. A larger " \ - "value allows a smaller number of mixed evacuations to process " \ + "a percent ratio. The default value 75 denotes that no more " \ + "than 75% of the collection set evacuation " \ + "workload may be evacuate to old-gen heap regions. This limits " \ + "both the promotion of aged regions and the compaction of " \ + "existing old regions. A value of 75 denotes that the normal " \ + "young-gen evacuation is increased by up to four fold. " \ + "A larger value allows quicker promotion and allows" \ + "a smaller number of mixed evacuations to process " \ "the entire list of old-gen collection candidates at the cost " \ "of an increased disruption of the normal cadence of young-gen " \ "collections. A value of 100 allows a mixed evacuation to " \ @@ -378,7 +364,7 @@ "to be less than this.") \ range(0, 100) \ \ - product(uintx, ShenandoahMaxYoungPercentage, 80, EXPERIMENTAL, \ + product(uintx, ShenandoahMaxYoungPercentage, 100, EXPERIMENTAL, \ "The maximum percentage of the heap to use for the young " \ "generation. Heuristics will not adjust the young generation " \ "to be more than this.") \ @@ -431,10 +417,14 @@ "When running in passive mode, this can be toggled to measure " \ "either Degenerated GC or Full GC costs.") \ \ - product(uintx, ShenandoahFullGCThreshold, 3, EXPERIMENTAL, \ + product(uintx, ShenandoahFullGCThreshold, 64, EXPERIMENTAL, \ "How many back-to-back Degenerated GCs should happen before " \ "going to a Full GC.") \ \ + product(uintx, ShenandoahOOMGCRetries, 3, EXPERIMENTAL, \ + "How many GCs should happen before we throw OutOfMemoryException "\ + "for allocation request, including at least one Full GC.") \ + \ product(bool, ShenandoahImplicitGCInvokesConcurrent, false, EXPERIMENTAL, \ "Should internally-caused GC requests invoke concurrent cycles, " \ "should they do the stop-the-world (Degenerated / Full GC)? " \ @@ -518,20 +508,6 @@ "Fix references with load reference barrier. Disabling this " \ "might degrade performance.") \ \ - product(uintx, ShenandoahBorrowPercent, 30, EXPERIMENTAL, \ - "During evacuation and reference updating in generational " \ - "mode, new allocations are allowed to borrow from old-gen " \ - "memory up to ShenandoahBorrowPercent / 100 amount of the " \ - "young-generation content of the current collection set. " \ - "Any memory borrowed from old-gen during evacuation and " \ - "update-references phases of GC will be repaid from the " \ - "abundance of young-gen memory produced when the collection " \ - "set is recycled at the end of updating references. The " \ - "default value of 30 reserves 70% of the to-be-reclaimed " \ - "young collection set memory to be allocated during the " \ - "subsequent concurrent mark phase of GC.") \ - range(0, 100) \ - \ product(uintx, ShenandoahOldCompactionReserve, 8, EXPERIMENTAL, \ "During generational GC, prevent promotions from filling " \ "this number of heap regions. These regions are reserved " \ diff --git a/test/hotspot/jtreg/ProblemList.txt b/test/hotspot/jtreg/ProblemList.txt index 546255a5b6c85..64a58d5cdaa30 100644 --- a/test/hotspot/jtreg/ProblemList.txt +++ b/test/hotspot/jtreg/ProblemList.txt @@ -83,11 +83,6 @@ gc/stress/gclocker/TestGCLockerWithG1.java 8180622 generic-all gc/stress/TestJNIBlockFullGC/TestJNIBlockFullGC.java 8192647 generic-all gc/stress/TestStressG1Humongous.java 8286554 windows-x64 -gc/shenandoah/oom/TestThreadFailure.java 8306335 generic-all -gc/shenandoah/oom/TestClassLoaderLeak.java 8306336 generic-all -gc/stress/gclocker/TestGCLockerWithShenandoah.java#generational 8306341 generic-all -gc/TestAllocHumongousFragment.java#generational 8306342 generic-all - ############################################################################# # :hotspot_runtime From 0c62f9cbcda9b61d9d7cc301f3bcc3a529b1194d Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Fri, 26 May 2023 21:10:18 +0000 Subject: [PATCH 239/254] Remove early planning docs from PR --- genshen-docs/README.md | 9 - genshen-docs/design.summary.md | 329 ------------------------- genshen-docs/glossary.md | 44 ---- genshen-docs/workplan.summary.md | 395 ------------------------------- 4 files changed, 777 deletions(-) delete mode 100644 genshen-docs/README.md delete mode 100644 genshen-docs/design.summary.md delete mode 100644 genshen-docs/glossary.md delete mode 100644 genshen-docs/workplan.summary.md diff --git a/genshen-docs/README.md b/genshen-docs/README.md deleted file mode 100644 index ec077704b35ce..0000000000000 --- a/genshen-docs/README.md +++ /dev/null @@ -1,9 +0,0 @@ -## GenShen: the Generational Shenandoah project - -Gen Shen stands for Generational Shenandoah, -and it also means "deep roots" in Mandarin. - -Documents in this directory: -- [glossary.md](glossary.md): glossary of terms used in our documents, -- [workplan.summary.md](workplan.summary.md): summary of the plan of record that we strive to follow, -- [design.summary.md](design.summary.md): summary of design points that we rely upon as working hypotheses. diff --git a/genshen-docs/design.summary.md b/genshen-docs/design.summary.md deleted file mode 100644 index 22393046c0a42..0000000000000 --- a/genshen-docs/design.summary.md +++ /dev/null @@ -1,329 +0,0 @@ -## Design Points - -This section discusses design decisions, as guided by the following tenets. - -### Tenets - -1. Don’t punish the mutator: While mutator code must be added to - implement generational collection, degradations of - mutator throughput and responsiveness - should be contained as much as possible. - If in doubt between design choices, - pick the one that promises to minimize mutator overhead. -2. Safeguard incremental progress with milestones that demonstrate key capabilities. -3. Minimize the overall development effort. This includes containing - efforts to prepare and present each milestone deliverable - that do not directly contribute to the end product. - -### Design Decisions - -While it is probable that we may revisit some of the following design -decisions if we run into implementation difficulties or performance -problems, the following is the current plan of record. - -1. This document is maintained alongside the source code. - 1. It is organized hierarchically with sections - dedicated to major design decisions. - Further, more detailed decisions are organized in top-down fashion, - with overarching design decisions preceding derivative design decisions. - The topic numbers assigned to particular design decisions may evolve over time. - 2. There is a separate "rationale" document that explains more of the reasoning behind design decisions - and links relevant portions between documents. -2. Genshen changes to the Shenandoah implementation will be - compartmented. See discussion [here](rationale.summary.md#compartmentalization"). -3. We will support concurrent collection of old gen and young gen, - with the expectation that a single old collection will typically - overlap with the execution of many young collections. -4. In order to minimize performance impact on the mutator, we will use - a single Load-Reference-Barrier to implement both evacuation - from young gen and from old gen. - 1. Tenuring will be implemented as part of young evacuation. - 2. All evacuation for both young and old gen regions - happens under the control of the same GC threads and is - supported by the same Load Reference Barrier (LRB) as in - vanilla Shenandoah, with only small refinements to the - implementation of slow-path code within the LRB. - 3. See discussion [here](rationale.summary.md#load-reference-barrier). -5. An old collection begins at the same time as a young gen - collection, with both collections leveraging the - results of a single shared root scan. -6. Old collection generally runs at lower priority (e.g., fewer - numbers of parallel threads or greater “nice” values for - concurrent threads) than young collection because - replenishing the allocation pool to support ongoing allocation - needs of mutator threads requires urgent completion of young GC. - See discussion [here](rationale.summary.md#prioritization). -7. Although the planned future production release of GenShen will - run young collection concurrently with old collection, support - will also be implemented and maintained for alternating executions - of young collections with global collections. See - discussion [here](rationale.summary.md#young-global-discussion). - 1. Since this is not a production release, efficiency of - implementation is not a primary concern. - 2. Regression tests will exercise the implementation of this mode - of operation. -8. A regression test suite will be developed and maintained to - support all of the enhanced capabilities of GenShen, including - capabilities that are enabled and disabled at build time for the - JVM. -9. Though old collections start at the same time as young - collections collections, they do not necessarily end at the same - time. Typically, many young-gen collections will be completed - concurrently during the time that a single old-gen collection is - running. See discussion [here](rationale.summary.md#concurrency-of-young-and-old). -10. Root scanning will be performed during a JVM safe point. -11. Scanning of the remembered set will be performed by parallel - young-gen GC threads during a JVM safe point. A side - effect of scanning is to mark as CLEAN any cards of the remembered - set that are found by remembered set scanning to no longer be - DIRTY. -12. The remembered set maintains a start-of-object representation to - facilitate quick identification of where the first object - pertaining to each DIRTY card region begins. See discussion - [here](rationale.summary.md#remembered-set-starting-offsets). - This requires that: - 1. Newly promoted objects be registered with the remembered set - and start-of-object support by post-processing the associated GCLAB block, and - 2. When entire regions are promoted out of young collections into - old gen, the objects contained within those regions must be - registered with the remembered set and the start-of-object support. -13. Young marking, including scanning of root pointers, - will place discovered references to old gen into a thread-local SATB - buffer so that they can be processed by an old-gen collection - thread. See discussion [here](rationale.summary.md#satb-keep-old-alive-entries). - 1. Possible optimization: refrain from logging old-gen pointers - that refer to already marked old objects into the SATB buffer. -14. The default size of the SATB buffer will increase from 1024 to - 4096 words because we will be placing more information into the - SATB buffer. -15. Whenever the SATB buffer is full, the slow path for adding to - the SATB buffer will attempt to compress the buffer contents before - communicating the buffer to GC threads. If compression leaves a - minimum of 256 slots available within the SATB buffer, the thread - continues to add values to the existing buffer. Compression - consists of: - 1. For pointers to young gen: - 1. If concurrent marking of young gen is disabled, ignore the - pointer. - 2. Otherwise, if the object referenced by this pointer has - already been marked, ignore the pointer. - 3. If we choose to use SATB-based remembered set, ignore all - overwritten address values that reside within young gen. - 2. For pointers to old gen: - 1. If concurrent marking of old gen is disabled, ignore the - pointer. - 2. Otherwise, if the object referenced by this pointer - has already been marked, ignore the pointer. - 3. If we choose to use an SATB-based remembered set: - - If the card corresponding to an overwritten old gen - address is already DIRTY, ignore this overwritten - address. - - Otherwise, fetch the pointer value held at the overwritten - address. If the fetched pointer value does not - refer to young gen, ignore this overwritten address. - - Otherwise, use a hash table (suggested size 127) to sift - redundant DIRTY card references for the current batch of - overwritten old-gen addresses. Preserve only one address - for each card entry that needs to be marked as DIRTY. - 3. SATB buffers will be processed by both a young GC - thread and an old GC thread. - 1. The young GC thread will mark objects referenced - by young pointers. - 2. The old GC thread will: - 1. Mark objects referenced by old-gen pointers, and - 2. If we choose to use SATB-based remembered set: mark as DIRTY - the card entries corresponding to overwritten addresses. -16. GC uses a G1-style heuristic to choose collection sets for both - young and old memory areas. The collection set represents the - heap regions that offer the greatest opportunity to quickly reclaim - garbage, as with the existing Shenandoah implementation. See - discussion [here](rationale.summary.md#g1-heuristic). -17. Following an evacuation phase that evacuates - both old and young heap regions, the update-references phase is - required to update references throughout all - old regions that were not selected as part of the old - collection set in addition to updating references in young - heap regions. - 1. If a particular young collection evacuation phase does not - evacuate any old regions, then its update references phase can - focus solely on young heap regions and the - remembered set. -18. Tenuring is based on object age with the enhancements described - below. See discussion [here](rationale.summary.md#tenuring). - 1. The variable TenureCycle counts how many GC cycles correspond to - each increment of an object’s age. Object ages are not - necessarily incremented each time a GC cycle is completed. They - are incremented each time TenureCycle GC cycles have been - completed. - 2. The variable TenureAge represents the age at which an object - becomes eligible for tenuring. - 3. During GC cycles that correspond to TenureCycle, the “age” of - individual objects is incremented by 1 plus the size of the - object’s original heap region age when the object is evacuated. - 4. During GC cycles that correspond to TenureCycle, the “age” of - each heap region that has been fully allocated (i.e. no more - available memory for allocation) and that is not in the - collection set is incremented by 1 at the end of the evacuation - phase. If the resulting “age” equals TenureAge, then the entire - region is reassigned to become part of old gen. - 1. The update-refs phase will process this heap region even - though it is “now” considered to be part of old gen. - 2. Each of the objects contained within the promoted region - shall be registered with the remembered set abstraction. - 3. Each of the objects contained within the promoted region - be scanned to determine any references to young-gen - memory that are contained therein. For any such pointers, set - the associated card table entry to DIRTY. -19. During evacuation, each running mutator thread has both a TLAB - and a GCLAB. - 1. The TLAB is associated with a young heap region. - 2. The GCLAB is associated with an old heap region. - 3. When the mutator’s load reference barrier encounters a - reference for which the associated object needs to be tenured, it - allocates the copy memory from the GCLAB. - 4. When the mutator’s load reference barrier encounters a - reference for which the associated object resides within the - collection set of old gen, it allocates the copy memory from the - GCLAB. - 5. When the mutator’s load reference barrier encounters a - reference for which the associated object needs to be evacuated to - young gen, it allocates the copy memory from the TLAB. - 6. We initially plan to use the same size for TLAB and GCLAB, but - this decision may be revisited. - 7. If the size of the object to be evacuated is larger than half - the size of the respective local allocation buffer, allocation - of the replica memory is handled by alternative allocators, to be - designed. Call these "odd" objects. -20. During evacuation, each evacuating GC thread will maintain two - GCLAB buffers: - 1. GCLAB-Young is associated with young gen. - 2. GCLAB-Old is associated with old gen. - 3. If the object to be evacuated currently resides in old gen or - if it resides in young gen and it is to be tenured, allocate the - copy memory from GCLAB-Old. - 4. Otherwise, allocate the copy memory from GCLAB-Young. - 5. At the end of the evacuation phase, consider repurposing any - unspent GCLAB-Young as a TLAB if there is sufficient unallocated - memory remaining within it. - 6. At the end of the evacuation phase, consider preserving the - GCLAB-Old for use as a GCLAB for a mutator thread during the next - young collections collection or as a GCLAB-Old during the next - old-gen evacuation pass. - 7. See 19.7. -21. A certain budget of CPU time is provisioned to perform young-gen - GC in order to support a particular planned workload. - 1. The resources dedicated to young-gen GC are limited in order - to assure a certain amount of CPU time is available to mutator - threads. - 2. Command line options allow user control over provisioning. A - TBD API may allow services to adjust provisioning dynamically. - 3. In the ideal, provisioning is adjusted automatically based on - TBD heuristics. - 4. The provisioned CPU resources can support a range of service - quality, reclaiming large numbers of heap regions with a low GC - frequency or reclaiming small numbers of heap regions with a - high GC frequency. Given a particular frequency of GC cycles, - the same CPU resources can evacuate a large number of sparsely - populated heap regions or a small number of densely populated - heap regions. Tradeoffs between these configuration extremes - may be adjusted under software control or by TBD - heuristics. - 5. For each young-gen GC pass, a certain TBD percentage - of CPU resources are reserved for old-gen evacuation and - update-refs activities. - 1. The old-gen CPU resource budget represents the total amount - of old-gen memory that can be relocated, and is quantified as - a multiple N of the heap region size. - 2. The old-gen GC threads determine the composition of the - old-gen collection set, up to but never exceeding the upper - bound N on cumulative evacuation size. - 3. The old-gen GC thread may select for the collection set - N heap regions which are known to have 100% - utilization, 2N heap regions known to have 50% utilization, - 5N heap regions known to have 20% utilization, and so on. - 4. If the old-gen GC refrains from delegating N heap regions - worth of evacuation work to the young-gen evacuation phase, - then the young GC is free to use the excess CPU resources to - more aggressively evacuate more of its own young heap regions, - using a larger than normal young-gen collection set. - 5. The budget for updating old-gen references must be large - enough to handle the total old-gen memory size - N. In the - case that old-gen GC configures a collection set that - represents its full evacuation capacity of N heap regions, the - number of old-gen heap regions that are not part of the - old-gen collection set is never larger than this quantity. - In the case that old-gen GC configures a smaller collection - set, then for each fraction of a heap region that is not - evacuated, this much more of a heap region might have to be - processed during the update-refs phase of GC. We estimate - that the cost of evacuating M bytes of memory is similar to - the cost of updating the references within M bytes of - memory. -22. A smaller (generally) budget of CPU time is provisioned to - perform old-gen GC in order to support a particular planned - workload. - 1. The resources dedicated to young-gen GC are limited in order - to assure a certain amount of CPU time is available to mutator - threads. - 2. Command-line options allow user control over provisioning. A - TBD API may allow services to adjust provisioning dynamically. - 3. In the ideal, provisioning is adjusted automatically based on - TBD heuristics. - 4. As with young-gen GC, the CPU resources provisioned for - old-gen GC can support a range of service quality. - 5. The CPU resources dedicated to old-gen collection do not have - responsibility for evacuating old regions as all evacuation - is performed by young-gen GC threads. Instead, the CPU - resources dedicated to old-gen GC activities are used for - activities such as the following: - 1. Processing the content of SATB buffers: - - If old collection is in concurrent marking phase, mark - objects referenced by any keep-alive pointers. - - If we are using SATB-based remembered set, update the - remembered set based on overwritten addresses reported in the - SATB buffer. - 2. During concurrent marking, scan the contents of previously - marked objects to complete the transitive closure of - reachability. - 3. Perform heuristic computations: - - Determine when to start the next old-gen GC cycle. - - Determine which old regions to evacuate on this and future - passes of young-gen GC. - - Adjust young-gen efficiency parameters such as: How many - heap regions should be dedicated to young gen? What is - optimal value of TenureCycle? What is optimal value of - TenureAge? How much CPU time should be dedicated to - young-gen GC? - - Adjust old-gen efficiency parameters such as: How much CPU - time should be dedicated to old-gen GC? How many heap regions - should be dedicated to old gen? Should any heap regions be - decommissioned and returned to the operating system in order - to shrink the memory footprint of this service? - 4. Perform routine maintenance as time and schedule permits: - - Potentially sweep up dead memory, accumulating ages at - which dead objects were reclaimed within old regions. - - Potentially, sweep through evacuated memory to accumulate - ages at which dead objects were reclaimed. - - Organize free lists for fast and efficient allocation of - GCLAB and Odd object allocations. - - Return emptied old regions to the free set. - - Eventually, reference processing and string deduplication - should be performed by lower priority old-gen threads - rather than higher priority young-gen threads. - 5. Following completion of each old-gen concurrent mark pass, - select regions to be included in the old-gen collection set: - - No more than a total of N bytes of old gen is evacuated by - each pass of the young-gen evacuator. - - If old-gen GC threads desire to evacuate M, which is more - than N bytes of old gen, it does so by piggy backing on - multiple subsequent young-gen evacuation passes, selecting - evacuating no more than the accumulation of N total heap - regions in each of the following young-gen evacuation passes. - - Since it is most efficient to evacuate all M > N regions - of old-gen memory with a single young-gen evacuation pass, - configure the old-gen collection set to include all M - regions if there are sufficient free regions available to - afford the young-gen allocator to continue allocating new - objects during the longer delay that would be required to - evacuate more than the traditionally budgeted N regions of - old-gen memory. diff --git a/genshen-docs/glossary.md b/genshen-docs/glossary.md deleted file mode 100644 index f8b98564557f0..0000000000000 --- a/genshen-docs/glossary.md +++ /dev/null @@ -1,44 +0,0 @@ -## Glossary for the Generational Shenandoah (GenShen) Project - -Shen := Shenandoah := the Shenandoah garbage collector - -GenShen := Generational Shenandoah - -gen := generation -young gen := the young generation -old gen := the old generation - -collector := garbage collector -young collector := young gen collector -old collector := old gen collector -global collector := single-generation collector that works on the entire heap - -young/old/global collection := a complete cycle through the phases of the young/old/global collector - -cset := collection set -remset := remembered set -rset := remembered set - -parallel := same task, dealt with by multiple threads -concurrent := different tasks, operated upon simultaneously -conc := concurrent - -conc mark := concurrent marking -conc global GC := concurrent global GC, like vanilla Shen -evac := evacuation (phase) -UR := update references (phase) - -LRB := Load Reference Barrier -SATB := Snapshot At The Beginning -TLAB := Thread-Local Allocation Buffer for a mutator thread -GCLAB := like a TLAB, but for a GC thread - -young region := a heap region affiliated with young gen -old region := a heap region affiliated with old gen -free region := a region that is not affiliated with either generation and available for future reuse by allocators - -young object := an object in young gen -old object := an object in old gen - -block := an identifiable chunk of space in a region that is or was occupied by a Java object -block start := a pointer to the beginning of a block diff --git a/genshen-docs/workplan.summary.md b/genshen-docs/workplan.summary.md deleted file mode 100644 index dde3c0c0d3b3d..0000000000000 --- a/genshen-docs/workplan.summary.md +++ /dev/null @@ -1,395 +0,0 @@ -# Summary Plan of Record: GenShen Prototype (2020) - -## Planned Milestones - -### Overview of Initial Milestones - -1. Pure young collection. (Young gen size unlimited. Old gen untouched.) -2. Size-restricted young gen. -3. Tenuring and promotion. -4. Global collection after young collection. -5. Young collection after global collection, repeat alternations. -6. Concurrent old marking. -7. Concurrent old and young collections. - -Young collection reclaims garbage only from heap regions that are -identified as belonging to the young generation. - -Old collection reclaims garbage only from heap regions that are -identified as belonging to the old generation, which holds objects -that have been promoted from young generation. - -Global collection reclaims garbage from heap regions belonging to -either the young generation or the old generation. - -### Milestone 1: GC of Young Gen Only - -This demonstration proves successful implementation of: - -1. Separating out that certain heap regions comprise young gen and - other heap regions are considered to represent old gen. -2. Confirming that card marking does not crash. (Does not prove that - card marking works because we have no objects in old gen.) -3. Confirming that remembered set scanning does not crash. (Does not - prove that remembered set scanning works because we have no objects in - old gen.) -4. Demonstrating a simplified form of young collection. - -Tasks - -1. Integrate various completed code patches into a common branch -2. Test and debug existing code - -### Milestone 2: Restrict Size of Young Gen - -This milestone constrains the young generation to a certain number of -regions. After that number is reached, allocation fails. For now, the -number can be fixed, say 25% of all regions, and we can simply crash -thereafter. - -Tasks - -1. Establish a mechanism to specify and enforce a limit on the number - of heap regions that may comprise young-gen memory. -2. Adjust the GC triggering mechanism so that the size of young gen - does not have reason to exceed the young-gen size during young-gen - GC, where the size of young-gen is affected by GC in the - following ways: - 1. Certain free heap regions may be added to young gen in order to - support allocation requests that are made by concurrent mutator - threads while GC is being performed. - 2. At the end of GC, all heap regions that were part of the - collection set are removed from young-gen memory and placed - in the free pool. - - -### Milestone 3: Young Collection with Promotion of Tenured Objects - -Add to Milestone 2 the capability of promoting young gen objects. -Don’t worry about odd objects or humongous objects at this time. -Demonstrate: - -1. That we can promote objects into old gen. -2. That card-marking works. -3. That remembered set scanning works (correctly, but perhaps not with - full desired efficiency). - -The following tasks must be implemented: - -1. The collection set is selected as a subset of all young collections - heap regions using TBD heuristics. -2. During young collection, each time an object is evacuated to a - young consolidation region, the object’s age is incremented. - 1. For simplicity, don’t try to implement the complete region - aging or promotion at this time. - 2. Also for simplicity, don’t try to implement the TenureCycle - optimization. -3. During young collection, each GC thread maintains both a young - GCLAB and and old GCLAB. - 1. When evacuating an object whose incremented age is less than - TenureAge, allocate memory for the object’s copy from within the - young GCLAB. - 2. When evacuating an object whose incremented age is >= - TenureAge, allocate memory for the object’s copy from within the - old GCLAB. - 3. Don’t support Odd or Humongous objects for simplicity. -4. During young collection, each mutator thread maintains both a TLAB - and a GCLAB. The GCLAB is associated with old gen. - 1. When evacuating an object whose incremented age is less than - TenureAge, allocate memory for the object’s copy from within the - TLAB. - 2. When evacuating an object whose incremented age is >= - TenureAge, allocate memory for the object’s copy from within the - GCLAB. -5. Perform maintenance on the contents of each retired old GCLAB, - where this maintenance consists of: - 1. Registering each object’s starting location and length with the - remembered set abstraction so that remembered set scanning can - quickly find the first object within each DIRTY card region, - 2. Updating the card table to reflect all references from this - object into young gen, - 3. Evacuating all objects directly referenced from this object - which reside within a collection set and have not yet been - evacuated, and - 4. Healing all pointers from within newly evacuated old objects - that refer to objects residing within the collection set. - 5. The last two items listed above are already performed by - traditional Shenandoah but can be merged with the implementation of - the other maintenance activities in order to perform all this work - in a consolidated pass. - -### Milestone 4: Sequential Execution of Multiple Young Collections Followed by Multiple Global Collections - -__Demonstrated Concurrency Between Young-Gen and Old-Gen Activities__ - - ✓ denotes that this combination of activities is allowed. - - ✗ denotes that this combination of activities is disallowed. - -| | Old-Gen Mark | Old-Gen Evac | Old-Gen Idle | -|:--------------:|:--------------:|:-------------:|:------------:| -| Young Gen Mark | ✓ | ✗ | ✓ | -| Young Gen Evac | ✗ | ✓ | ✓ | -| Young Gen Idle | ✗ | ✗ | ✓ | - -Add to Milestone 3 the ability to switch to global collection after a -series of young Collections. - -1. The switch is triggered by a simple TBD test, such as when the - space available within old gen is less than the size of young gen. -2. The global collection continues to run with the card-marking - barrier enabled though the card values will not be consulted further. -3. For this demonstration, the switch to global collections is - one-way. Following this switch, we can no longer perform young - collections. -4. For this demonstration, the global collection does not distinguish - between young regions and old regions. - 1. Evacuation of objects that resided in an old region is handled - the same as evacuation of objects that resided in a young heap - region. -5. This demonstrates that: - 1. Promotion of objects works. The objects that have been promoted - into old gen maintain whatever protocols and invariants are - assumed by Shenandoah GC. - 2. That we can manage the transition from young GC to global GC. - 3. That global collection, insofar as we characterize it, works. - 4. That Young collections do not corrupt the heap. - -Tasks: - -1. Fix bugs in existing code enhancements. -2. Implement the transition from young collection to global collection -3. Implement global collection with write barrier for card-table marking - -### Milestone 5: Interleaved Execution of Young and Global Collections - -__Demonstrated Concurrency Between Young-Gen and Old-Gen Activities__ - - ✓ denotes that this combination of activities is allowed. - - ✗ denotes that this combination of activities is disallowed. - -| | Old-Gen Mark | Old-Gen Evac | Old-Gen Idle | -|:--------------:|:--------------:|:-------------:|:------------:| -| Young Gen Mark | ✓ | ✗ | ✓ | -| Young Gen Evac | ✗ | ✓ | ✓ | -| Young Gen Idle | ✗ | ✗ | ✓ | - -Add to Milestone 4 the ability to switch back to Young Collection upon -completion of a global collection. - -1. Assume that the global collection is successful in reclaiming - necessary memory. No heuristics to resize oldgen or young gen at - this time. Specify sizes of each on command line. -2. The switch to old collection is triggered by exhaustion of old gen. - At least one young collection is assumed to execute following - completion of each global collection. -3. For this demonstration, the global collection does distinguish - between young collections and old regions. - 1. Objects in the old collection set are evacuated to old - consolidation regions. - 2. Objects in the young collection set are evacuated - to young collections consolidation regions. - 3. There is no tenuring of objects during a global collection (for - simplicity). - -This demonstrates that: - -1. Promotion of objects works. The objects that have been promoted - into old gen maintain whatever protocols and invariants are - assumed by Shenandoah GC. -2. That we can manage the transition from young GC to global GC. -3. That we can manage the transition from global GC to young GC. -4. That the transition from global GC to young GC - establishes all invariants required for correct operation of young - GC. - -Tasks: - -1. Distinguish between “global collection” for purposes of - maintaining support for non-generational GC and “global collection” - for purposes of supporting sequential interleaving of young GC - and global GC. -2. Implement the transition from global GC to young GC. -3. Initialize the remembered set for each consolidation heap region - of old gen to all CLEAN before allocating any GCLAB buffers within - the consolidation heap region. -4. At the start of global evacuation, select the collection set as - some combination of existing young regions and - old regions based on heuristics TBD. -5. During global collection, maintain old-gen GCLABs for all GC - threads and mutator threads. -6. During global collection, distinguish evacuation behavior - depending on whether an object to be evacuated resides within the - young collection set or the old collection set since - young objects are evacuated into young - consolidation regions and old objects are evacuated into old - consolidation regions; -7. Add minimal logging reports to describe behavior of young-gen and global - GC. -8. During global collection, perform maintenance on the contents of - each retired old GCLAB, where this maintenance consists of: - 1. Registering each object’s starting location and length with the - remembered set abstraction so that remembered set scanning can - quickly find the first object within each DIRTY card region, - 2. Updating the card table to reflect all references from this - object into young gen, - 3. Evacuating all objects directly referenced from this object - that reside within a collection set and that have not already been - evacuated, and - 4. Healing all pointers from within new replica objects residing in old - gen that refer to objects residing within the collection set. - 5. The last two items listed above are already performed by - traditional Shenandoah but can be merged with the implementation of - the other maintenance activities in order to perform all this work - in a consolidated pass. - -### Milestone 6: GC of young collections with Concurrent Marking (but not Collecting) of Old Gen - -__Demonstrated Concurrency Between Young-Gen and Old-Gen Activities__ - - ✓ denotes that this combination of activities is allowed. - - ✗ denotes that this combination of activities is disallowed. - -| | Old-Gen Mark | Old-Gen Evac | Old-Gen Idle | -|:--------------:|:--------------:|:-------------:|:------------:| -| Young Gen Mark | ✓ | NA | ✓ | -| Young Gen Evac | ✓ | NA | ✓ | -| Young Gen Idle | ✓ | NA | ✓ | - -This demonstration relies on GC log reports to show that marking of -old gen runs concurrently with marking of young gen. -Since the results of old-gen marking are not used to support old-gen -evacuation, this demonstration does not prove that old-gen marking -produces correct results. - -All pointers to old-gen memory that are discovered during scan of -young-gen memory are communicated to the old-gen concurrent mark -threads by inserting these pointer values into a SATB buffer as -keep-alive values. Every SATB buffer is post-processed both by a -young-gen GC thread and by an old-gen GC thread. - -Pointers from old-gen memory to young-gen memory that are discovered -during the marking of old-gen are ignored. - -At the start of young-gen concurrent marking, the remembered set is -scanned to detect all inter-generational references. - -The SATB write barrier remains enabled as long as either young-gen or -old-gen concurrent marking is active. - -Tasks: - -1. Each young GC thread has a dedicated SATB buffer into which it places - discovered references to old-gen memory. -2. SATB write barrier is left enabled as long as either young - or old marking is active. -3. SATB buffer is enlarged to 4096 entries. -4. SATB buffer compression is enhanced to deal with the mix of old - and young pointers. -5. SATB buffer processing is performed by both a young collection - thread and an old collection thread. Pointers to - old gen within the SATB buffer are marked and added to the old-gen - closure queues so that they can be subsequently scanned. -6. Certain GC threads (or work items) are dedicated to old GC - and others are dedicated to young GC. -7. Old-gen GC threads process the closure of previously marked - old-gen objects, scanning all references contained therein. -8. Old-gen GC threads add to the old-gen closures all old-gen objects - referenced by SATB buffers if those objects were not previously marked. - -### Milestone 7: Concurrent Young and Old Collections - - -__Demonstrated Concurrency Between Young-Gen and Old-Gen Activities__ - - ✓ denotes that this combination of activities is allowed. - - ✗ denotes that this combination of activities is disallowed. - -| | Old-Gen Mark | Old-Gen Evac | Old-Gen Idle | -|:--------------:|:--------------:|:-------------:|:------------:| -| Young Gen Mark | ✓ | ✗ | ✓ | -| Young Gen Evac | ✓ | ✓ | ✓ | -| Young Gen Idle | ✓ | ✗ | ✓ | - -In this demonstration, old-gen concurrent marking runs concurrently with all -phases of young-gen GC. Old-gen evacuation only runs while young-gen -evacuation is running. In the case that old-gen needs to evacuate so -much memory that doing so in a single uninterruptible batch would -significantly extend the duration of the young-gen evacuation phase, -the total old-gen evacuation workload is divided into multiple smaller -batches of evacuation work, each batch being processed concurrently -with a different young-gen evacuation cycle. - -The demonstration: - -1. Uses logging reports to describe the results of young - collection and old collection. -2. Shows for some “simple” workload (Heapothysis or - Extremem?) that generational GC provides performance benefits over - non-generational GC. - -Tasks - -1. Add minimal logging reports to describe behavior of old-gen - GC. -2. Decide which old regions comprise the old collection set. -3. Divide the old collection set into multiple collection subsets. -4. For each of the collection subsets - 1. Communicate the subset to young GC tasks to process these - evacuations when it begins its next evacuation cycle. - 2. Wait for young GC tasks to signal completion of the evacuation - cycle. - -## Proposed Future Milestones Not Yet Fully Planned - -### Milestone 8: Performance Improvements - -1. Remembered Set scanning sets cards to CLEAN if they are no longer - DIRTY. -2. Remembered Set scanning maintains and utilizes start-offset data - structure to quickly find the first object to be scanned within each - DIRTY card. -3. Remembered set scanning refrains from scanning the portions of - large objects and arrays that overlap card regions that are not - DIRTY. - -### Milestone 9: Fix Known Bugs - -We are aware of bugs in our existing card-marking implementation. - -### Milestone 10: Multiple Young-Gen Evacuations Process Old Collection Set - -### Milestone 11: Odd Objects (larger than 50% of TLAB/GCLAB size) - -By default, the promotion of such objects is handled by a -slower-than-normal path. Instead of allocating old gen from the -GCLAB, the mutator thread obtains memory for the copy by directly -accessing free lists. See existing code that does that already. - -### Micro Milestone 12: Collect and Report New Metrics - -### Micro Milestone 13: SATB-Based Remembered Set - -### Micro Milestone 14: Heuristic Pacing of Young Collection Frequency - -### Micro Milestone 15: Heuristic Pacing of Old Collection Frequency - -### Micro Milestone 16: Heuristic Sizing of Young and Old Sizes - -### Micro Milestone 17: Heuristic Adjustments of Tenuring Strategies - -### Micro Milestone 18: Overlap Evacuation of Cycle N with Marking of Cycle N+1 - -### Micro Milestone 19: Humongous Objects - -### Micro Milestone 20: Reference Processing - -### Micro Milestone 21: String Dedup - -### Micro Milestone 22: Degenerated GC - -### Micro Milesones TBD: Various Performance Improvements - From aa85a9073e2a71d6bf920409e739d555f9dcf302 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Tue, 30 May 2023 17:11:12 +0000 Subject: [PATCH 240/254] Revert changes to jcheck configuration --- .jcheck/conf | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.jcheck/conf b/.jcheck/conf index 38c0242c86040..8993c274fe0c9 100644 --- a/.jcheck/conf +++ b/.jcheck/conf @@ -1,10 +1,10 @@ [general] -project=shenandoah +project=jdk jbs=JDK version=21 [checks] -error=author,committer,reviewers,executable,symlink,message,hg-tag,whitespace,problemlists +error=author,committer,reviewers,merge,issues,executable,symlink,message,hg-tag,whitespace,problemlists [repository] tags=(?:jdk-(?:[1-9]([0-9]*)(?:\.(?:0|[1-9][0-9]*)){0,4})(?:\+(?:(?:[0-9]+))|(?:-ga)))|(?:jdk[4-9](?:u\d{1,3})?-(?:(?:b\d{2,3})|(?:ga)))|(?:hs\d\d(?:\.\d{1,2})?-b\d\d) @@ -18,12 +18,18 @@ domain=openjdk.org files=.*\.cpp|.*\.hpp|.*\.c|.*\.h|.*\.java|.*\.cc|.*\.hh|.*\.m|.*\.mm|.*\.md|.*\.gmk|.*\.m4|.*\.ac|Makefile ignore-tabs=.*\.gmk|Makefile +[checks "merge"] +message=Merge + [checks "reviewers"] -committers=1 +reviewers=1 ignore=duke [checks "committer"] role=committer +[checks "issues"] +pattern=^([124-8][0-9]{6}): (\S.*)$ + [checks "problemlists"] dirs=test/jdk|test/langtools|test/lib-test|test/hotspot/jtreg|test/jaxp From eb656ec26d635d0dce4aa11d48afbe2a571b919f Mon Sep 17 00:00:00 2001 From: William Kemper <71722661+earthling-amzn@users.noreply.github.com> Date: Thu, 1 Jun 2023 11:02:34 -0700 Subject: [PATCH 241/254] Make the order of young/old collector checks consistent (#1) --- src/hotspot/share/gc/shared/gcConfiguration.cpp | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/hotspot/share/gc/shared/gcConfiguration.cpp b/src/hotspot/share/gc/shared/gcConfiguration.cpp index bf2d8b4908399..cd5c3b9f36bf2 100644 --- a/src/hotspot/share/gc/shared/gcConfiguration.cpp +++ b/src/hotspot/share/gc/shared/gcConfiguration.cpp @@ -52,7 +52,7 @@ GCName GCConfiguration::young_collector() const { return NA; } - if (UseZGC) { + if (UseZGC) { if (ZGenerational) { return ZMinor; } else { @@ -71,14 +71,6 @@ GCName GCConfiguration::old_collector() const { return ParallelOld; } - if (UseZGC) { - if (ZGenerational) { - return ZMajor; - } else { - return Z; - } - } - if (UseShenandoahGC) { #if INCLUDE_SHENANDOAHGC if (strcmp(ShenandoahGCMode, "generational") == 0) { @@ -88,6 +80,13 @@ GCName GCConfiguration::old_collector() const { return NA; } + if (UseZGC) { + if (ZGenerational) { + return ZMajor; + } else { + return Z; + } + } return SerialOld; } From 5bf6e7e0a4c139fd5e6ac42ecd0d5810480c17eb Mon Sep 17 00:00:00 2001 From: William Kemper <71722661+earthling-amzn@users.noreply.github.com> Date: Thu, 1 Jun 2023 19:24:27 -0700 Subject: [PATCH 242/254] Assert bounds only when allocations succeed, increase test timeouts (#2) --- src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp | 1 - test/hotspot/jtreg/gc/shenandoah/TestRetainObjects.java | 2 +- test/hotspot/jtreg/gc/shenandoah/TestSieveObjects.java | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp index c873973459704..ede9911b6181e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFreeSet.cpp @@ -1484,7 +1484,6 @@ void ShenandoahFreeSet::log_status() { HeapWord* ShenandoahFreeSet::allocate(ShenandoahAllocRequest& req, bool& in_new_region) { shenandoah_assert_heaplocked(); - _free_sets.assert_bounds(); // Allocation request is known to satisfy all memory budgeting constraints. if (req.size() > ShenandoahHeapRegion::humongous_threshold_words()) { diff --git a/test/hotspot/jtreg/gc/shenandoah/TestRetainObjects.java b/test/hotspot/jtreg/gc/shenandoah/TestRetainObjects.java index 36fbeac514dda..3adeff9441854 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestRetainObjects.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestRetainObjects.java @@ -134,7 +134,7 @@ * @summary Acceptance tests: collector can deal with retained objects * @requires vm.gc.Shenandoah * - * @run main/othervm -Xmx1g -Xms1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * @run main/othervm/timeout=300 -Xmx1g -Xms1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions * -XX:+UseShenandoahGC * -XX:-UseTLAB -XX:+ShenandoahVerify * TestRetainObjects diff --git a/test/hotspot/jtreg/gc/shenandoah/TestSieveObjects.java b/test/hotspot/jtreg/gc/shenandoah/TestSieveObjects.java index 06389a67e518d..c627368ca5661 100644 --- a/test/hotspot/jtreg/gc/shenandoah/TestSieveObjects.java +++ b/test/hotspot/jtreg/gc/shenandoah/TestSieveObjects.java @@ -144,7 +144,7 @@ * @requires vm.gc.Shenandoah * @library /test/lib * - * @run main/othervm/timeout=240 -Xmx1g -Xms1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions + * @run main/othervm/timeout=300 -Xmx1g -Xms1g -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions * -XX:+UseShenandoahGC * -XX:-UseTLAB -XX:+ShenandoahVerify * TestSieveObjects From d4d2f1cf224200826593b67eabc2e0730f5a03f9 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Fri, 2 Jun 2023 02:42:36 +0000 Subject: [PATCH 243/254] Force PLAB sizes to align on card-table size --- src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index eaa510099555e..2a5406a0289f4 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -1275,6 +1275,14 @@ HeapWord* ShenandoahHeap::allocate_new_gclab(size_t min_size, HeapWord* ShenandoahHeap::allocate_new_plab(size_t min_size, size_t word_size, size_t* actual_size) { + size_t unalignment = min_size % CardTable::card_size_in_words(); + if (unalignment != 0) { + min_size = min_size - unalignment + CardTable::card_size_in_words(); + } + unalignment = word_size % CardTable::card_size_in_words(); + if (unalignment != 0) { + word_size = word_size - unalignment + CardTable::card_size_in_words(); + } ShenandoahAllocRequest req = ShenandoahAllocRequest::for_plab(min_size, word_size); // Note that allocate_memory() sets a thread-local flag to prohibit further promotions by this thread // if we are at risk of infringing on the old-gen evacuation budget. From 8d80780a64d7058bffa27b269157596d937a6b22 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Sun, 4 Jun 2023 21:29:32 +0000 Subject: [PATCH 244/254] Remove three asserts making comparisons between atomic volatile variables Though changes to the volatile variables are individually protected by Atomic load and store operations, these asserts were not assuring atomic access to multiple volatile variables, each of which could be modified independently of the others. The asserts were therefore not trustworthy, as has been confirmed by more extensive testing. --- .../share/gc/shenandoah/shenandoahGeneration.cpp | 11 ----------- .../gc/shenandoah/shenandoahHeapRegion.inline.hpp | 7 ------- 2 files changed, 18 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp index 3606e2cf63216..caf4f36dc5dd4 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGeneration.cpp @@ -657,11 +657,6 @@ void ShenandoahGeneration::establish_usage(size_t num_regions, size_t num_bytes, void ShenandoahGeneration::increase_used(size_t bytes) { Atomic::add(&_used, bytes); - // This detects arithmetic wraparound on _used. Non-generational mode does not keep track of _affiliated_region_count - // TODO: REMOVE IS_GLOBAL() QUALIFIER AFTER WE FIX GLOBAL AFFILIATED REGION ACCOUNTING - assert(is_global() || ShenandoahHeap::heap()->is_full_gc_in_progress() || - (_used + _humongous_waste <= _affiliated_region_count * ShenandoahHeapRegion::region_size_bytes()), - "used cannot exceed regions"); } void ShenandoahGeneration::increase_humongous_waste(size_t bytes) { @@ -683,12 +678,6 @@ void ShenandoahGeneration::decrease_used(size_t bytes) { assert(is_global() || ShenandoahHeap::heap()->is_full_gc_in_progress() || (_used >= bytes), "cannot reduce bytes used by generation below zero"); Atomic::sub(&_used, bytes); - - // Non-generational mode does not maintain affiliated region counts - // TODO: REMOVE IS_GLOBAL() QUALIFIER AFTER WE FIX GLOBAL AFFILIATED REGION ACCOUNTING - assert(is_global() || ShenandoahHeap::heap()->is_full_gc_in_progress() || - (_affiliated_region_count * ShenandoahHeapRegion::region_size_bytes() >= _used), - "Affiliated regions must hold more than what is currently used"); } size_t ShenandoahGeneration::used_regions() const { diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp index c5823b11ddd6d..2ec56b044c8a3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeapRegion.inline.hpp @@ -157,13 +157,6 @@ inline void ShenandoahHeapRegion::increase_live_data_gc_words(size_t s) { inline void ShenandoahHeapRegion::internal_increase_live_data(size_t s) { size_t new_live_data = Atomic::add(&_live_data, s, memory_order_relaxed); -#ifdef ASSERT - size_t live_bytes = new_live_data * HeapWordSize; - size_t used_bytes = used(); - assert(live_bytes <= used_bytes, - "%s Region " SIZE_FORMAT " can't have more live data than used: " SIZE_FORMAT ", " SIZE_FORMAT " after adding " SIZE_FORMAT, - affiliation_name(), index(), live_bytes, used_bytes, s * HeapWordSize); -#endif } inline void ShenandoahHeapRegion::clear_live_data() { From 9811d2aae265b2bb25f0e4f304638522346429e0 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Tue, 6 Jun 2023 21:01:03 +0000 Subject: [PATCH 245/254] Remove an inappropriate copyright notice --- .../gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp index 847bd283df22a..2e7da1f1dd257 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.cpp @@ -1,6 +1,5 @@ /* * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved. - * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it From cc149904d76c78355fc994da171f0f21411e903f Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Tue, 6 Jun 2023 21:30:20 +0000 Subject: [PATCH 246/254] Exit during initialization on unsupported platforms --- .../share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp index 9c5bce4abe797..c754b54d0ed8d 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahGenerationalMode.cpp @@ -61,6 +61,10 @@ ShenandoahHeuristics* ShenandoahGenerationalMode::initialize_heuristics(Shenando vm_exit_during_initialization("Generational mode requires the (default) adaptive heuristic"); } +#if !(defined AARCH64 || defined AMD64 || defined IA32 || defined PPC64) + vm_exit_during_initialization("Shenandoah Generational GC is not supported on this platform."); +#endif + return new ShenandoahAdaptiveHeuristics(generation); } From 8f9e2a84966008b3b94bf0268d06d468834ed6fc Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Tue, 6 Jun 2023 23:16:44 +0000 Subject: [PATCH 247/254] Improve efficiency of card-size alignment calculations --- src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp index 2a5406a0289f4..ea7be6a7e29ec 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp @@ -1275,14 +1275,10 @@ HeapWord* ShenandoahHeap::allocate_new_gclab(size_t min_size, HeapWord* ShenandoahHeap::allocate_new_plab(size_t min_size, size_t word_size, size_t* actual_size) { - size_t unalignment = min_size % CardTable::card_size_in_words(); - if (unalignment != 0) { - min_size = min_size - unalignment + CardTable::card_size_in_words(); - } - unalignment = word_size % CardTable::card_size_in_words(); - if (unalignment != 0) { - word_size = word_size - unalignment + CardTable::card_size_in_words(); - } + size_t words_in_card = CardTable::card_size_in_words(); + size_t align_mask = ~(words_in_card - 1); + min_size = (min_size + words_in_card - 1) & align_mask; + word_size = (word_size + words_in_card - 1) & align_mask; ShenandoahAllocRequest req = ShenandoahAllocRequest::for_plab(min_size, word_size); // Note that allocate_memory() sets a thread-local flag to prohibit further promotions by this thread // if we are at risk of infringing on the old-gen evacuation budget. From f6c073a54ee350dfc0d24050019757a5df365258 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Wed, 7 Jun 2023 00:33:43 +0000 Subject: [PATCH 248/254] Update copyright notices --- .../cpu/ppc/gc/shenandoah/c1/shenandoahBarrierSetC1_ppc.cpp | 3 +-- .../ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp | 2 +- .../ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.hpp | 1 - .../riscv/gc/shenandoah/c1/shenandoahBarrierSetC1_riscv.cpp | 1 - src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp | 1 - src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp | 1 - 6 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/hotspot/cpu/ppc/gc/shenandoah/c1/shenandoahBarrierSetC1_ppc.cpp b/src/hotspot/cpu/ppc/gc/shenandoah/c1/shenandoahBarrierSetC1_ppc.cpp index 6f2dd54deff96..79d1c9449b3fd 100644 --- a/src/hotspot/cpu/ppc/gc/shenandoah/c1/shenandoahBarrierSetC1_ppc.cpp +++ b/src/hotspot/cpu/ppc/gc/shenandoah/c1/shenandoahBarrierSetC1_ppc.cpp @@ -1,7 +1,6 @@ /* * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. - * Copyright (c) 2012, 2021 SAP SE. All rights reserved. - * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. + * Copyright (c) 2012, 2022 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp b/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp index 675427134f8b9..552ca1c6bc268 100644 --- a/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp +++ b/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp @@ -1,6 +1,6 @@ /* * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. - * Copyright (c) 2012, 2021 SAP SE. All rights reserved. + * Copyright (c) 2012, 2022 SAP SE. All rights reserved. * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * diff --git a/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.hpp b/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.hpp index e784cd1dde2eb..4b59ef8f6431f 100644 --- a/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.hpp +++ b/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.hpp @@ -1,7 +1,6 @@ /* * Copyright (c) 2018, 2022, Red Hat, Inc. All rights reserved. * Copyright (c) 2012, 2022 SAP SE. All rights reserved. - * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/cpu/riscv/gc/shenandoah/c1/shenandoahBarrierSetC1_riscv.cpp b/src/hotspot/cpu/riscv/gc/shenandoah/c1/shenandoahBarrierSetC1_riscv.cpp index 5334f3d165ec5..cd568cc723fe9 100644 --- a/src/hotspot/cpu/riscv/gc/shenandoah/c1/shenandoahBarrierSetC1_riscv.cpp +++ b/src/hotspot/cpu/riscv/gc/shenandoah/c1/shenandoahBarrierSetC1_riscv.cpp @@ -1,7 +1,6 @@ /* * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved. * Copyright (c) 2020, 2021, Huawei Technologies Co., Ltd. All rights reserved. - * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp index 994ec5a3719cc..0e811e6a2b2c3 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.cpp @@ -1,6 +1,5 @@ /* * Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved. - * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp index df7d6e685b9a3..65ddd8b1f118e 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahVMOperations.hpp @@ -1,6 +1,5 @@ /* * Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved. - * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it From 221c88ff8926038fd037537fc05bfc576acee0f9 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Wed, 7 Jun 2023 12:30:38 +0000 Subject: [PATCH 249/254] Remove one more extraneous Amazon copyright --- .../cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp b/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp index 552ca1c6bc268..21d6e2292f86e 100644 --- a/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp +++ b/src/hotspot/cpu/ppc/gc/shenandoah/shenandoahBarrierSetAssembler_ppc.cpp @@ -1,7 +1,6 @@ /* * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. * Copyright (c) 2012, 2022 SAP SE. All rights reserved. - * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it From 88958669d3f6c60bb6d115cc4e345f7ac1a2686e Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Wed, 7 Jun 2023 12:37:53 +0000 Subject: [PATCH 250/254] JDK-8309322: [GenShen] TestAllocOutOfMemory#large failed When generational Shenandoah is used, there may be an additional alignment related heap size adjustment that the test should be cognizant of. Such alignment might also happen in the non-generational case, but in this case the specific size used in the test was affected on machines with larger than usual os page size settings. The alignment related adjustment would have affected all generational collectors (except perhaps Gen Z). In the future, we might try and relax this alignment constraint.alignment. --- .../shenandoah/oom/TestAllocOutOfMemory.java | 38 +++++++++++++------ 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/test/hotspot/jtreg/gc/shenandoah/oom/TestAllocOutOfMemory.java b/test/hotspot/jtreg/gc/shenandoah/oom/TestAllocOutOfMemory.java index 7a7502f2c3d16..93d9d3abbc8dd 100644 --- a/test/hotspot/jtreg/gc/shenandoah/oom/TestAllocOutOfMemory.java +++ b/test/hotspot/jtreg/gc/shenandoah/oom/TestAllocOutOfMemory.java @@ -45,6 +45,7 @@ * @library /test/lib * @run driver TestAllocOutOfMemory small */ + import jdk.test.lib.process.OutputAnalyzer; import jdk.test.lib.process.ProcessTools; @@ -60,16 +61,16 @@ public static void work(int size, int count) throws Exception { } } - private static void allocate(String size) throws Exception { + private static void allocate(String size, int multiplier) throws Exception { switch (size) { case "large": - work(1024 * 1024, 16); + work(1024 * 1024, 16 * multiplier); break; case "heap": - work(16 * 1024 * 1024, 1); + work(16 * 1024 * 1024, multiplier); break; case "small": - work(1, 16 * 1024 * 1024); + work(1, 16 * 1024 * 1024 * multiplier); break; default: throw new IllegalArgumentException("Usage: test [large|small|heap]"); @@ -77,41 +78,56 @@ private static void allocate(String size) throws Exception { } public static void main(String[] args) throws Exception { - if (args.length > 1) { - // Called from test, size is second argument + if (args.length > 2) { + // Called from test, size is second argument, heap requested is third String size = args[1]; - allocate(size); + long spec_heap = Integer.parseInt(args[2]); + + // The actual heap we get may be larger than the one we asked for + // (particularly in the generational case) + final long actual_heap = Runtime.getRuntime().maxMemory(); + int multiplier = 1; + if (actual_heap > spec_heap) { + // A suitable multiplier is used, so as to allocate an + // amount appropriate to the larger actual heap size than what + // was specified. + multiplier = (int)((actual_heap + spec_heap - 1)/spec_heap); + } + + allocate(size, multiplier); return; } // Called from jtreg, size is first argument String size = args[0]; { + int heap = 16*1024*1024; // -Xmx16m expectFailure("-Xmx16m", "-XX:+UnlockExperimentalVMOptions", "-XX:+UseShenandoahGC", TestAllocOutOfMemory.class.getName(), - "test", size); + "test", size, Integer.toString(heap)); expectFailure("-Xmx16m", "-XX:+UnlockExperimentalVMOptions", "-XX:+UseShenandoahGC", "-XX:ShenandoahGCMode=generational", TestAllocOutOfMemory.class.getName(), - "test", size); + "test", size, Integer.toString(heap)); } { + int heap = 1*1024*1024*1024; // -Xmx1g expectSuccess("-Xmx1g", "-XX:+UnlockExperimentalVMOptions", "-XX:+UseShenandoahGC", TestAllocOutOfMemory.class.getName(), - "test", size); + "test", size, Integer.toString(heap)); expectSuccess("-Xmx1g", "-XX:+UnlockExperimentalVMOptions", "-XX:+UseShenandoahGC", "-XX:ShenandoahGCMode=generational", TestAllocOutOfMemory.class.getName(), - "test", size); + "test", size, Integer.toString(heap)); } } From 8e5c3b73996bc39763c55964600c69d3c03b6bbc Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Wed, 7 Jun 2023 13:29:47 +0000 Subject: [PATCH 251/254] Remove more extraneous copyright notices --- src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.hpp | 1 - src/hotspot/share/gc/shenandoah/shenandoahFullGC.hpp | 1 - src/hotspot/share/gc/shenandoah/shenandoahGC.cpp | 1 - src/hotspot/share/gc/shenandoah/shenandoahGC.hpp | 1 - src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp | 1 - src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.hpp | 1 - src/hotspot/share/gc/shenandoah/shenandoahSTWMark.hpp | 1 - src/hotspot/share/gc/shenandoah/shenandoahUnload.cpp | 1 - src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.cpp | 1 - src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.hpp | 1 - 10 files changed, 10 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.hpp index 32d26661ad076..085e77a0e52d6 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahDegeneratedGC.hpp @@ -1,6 +1,5 @@ /* * Copyright (c) 2021, Red Hat, Inc. All rights reserved. - * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.hpp index 4c28684c4050b..6687116b21f78 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahFullGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahFullGC.hpp @@ -1,6 +1,5 @@ /* * Copyright (c) 2014, 2021, Red Hat, Inc. All rights reserved. - * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGC.cpp b/src/hotspot/share/gc/shenandoah/shenandoahGC.cpp index 767465475df99..b0731b618f782 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGC.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGC.cpp @@ -1,6 +1,5 @@ /* * Copyright (c) 2021, Red Hat, Inc. All rights reserved. - * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahGC.hpp b/src/hotspot/share/gc/shenandoah/shenandoahGC.hpp index 0caac37c959f8..4e929363c947a 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahGC.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahGC.hpp @@ -1,6 +1,5 @@ /* * Copyright (c) 2021, Red Hat, Inc. All rights reserved. - * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp b/src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp index cf816563c64d7..cce02c2b194ba 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahNMethod.cpp @@ -1,6 +1,5 @@ /* * Copyright (c) 2019, 2022, Red Hat, Inc. All rights reserved. - * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.hpp b/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.hpp index 8dc75d0f3acbb..68f3cfba97af1 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahNumberSeq.hpp @@ -1,6 +1,5 @@ /* * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved. - * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.hpp b/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.hpp index 694cca0c005a7..265f404f7b8b2 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahSTWMark.hpp @@ -1,6 +1,5 @@ /* * Copyright (c) 2021, Red Hat, Inc. All rights reserved. - * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahUnload.cpp b/src/hotspot/share/gc/shenandoah/shenandoahUnload.cpp index 1b7e598b62f53..6072f605ea41c 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahUnload.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahUnload.cpp @@ -1,6 +1,5 @@ /* * Copyright (c) 2019, 2021, Red Hat, Inc. All rights reserved. - * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.cpp b/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.cpp index d1409851e3ec5..3ea4e17ca60fd 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.cpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.cpp @@ -1,6 +1,5 @@ /* * Copyright (c) 2017, 2019, Red Hat, Inc. All rights reserved. - * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.hpp b/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.hpp index ce1384a836d80..489be9723dd83 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahWorkerPolicy.hpp @@ -1,6 +1,5 @@ /* * Copyright (c) 2017, 2022, Red Hat, Inc. All rights reserved. - * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it From 01c625163f9c32b255adab232ae0fead5a014f52 Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Wed, 7 Jun 2023 14:38:16 +0000 Subject: [PATCH 252/254] Remove a few more unneeded copyright notices --- src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp | 1 - src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp | 1 - .../gc/shenandoah/heuristics/shenandoahCompactHeuristics.hpp | 1 - .../gc/shenandoah/heuristics/shenandoahPassiveHeuristics.hpp | 1 - .../gc/shenandoah/heuristics/shenandoahStaticHeuristics.hpp | 1 - src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.hpp | 1 - src/hotspot/share/gc/shenandoah/mode/shenandoahSATBMode.cpp | 1 - .../share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp | 1 - src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp | 1 - 9 files changed, 9 deletions(-) diff --git a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp index 20d0cf35c7532..8b82152973b31 100644 --- a/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp +++ b/src/hotspot/share/gc/shenandoah/c1/shenandoahBarrierSetC1.hpp @@ -1,6 +1,5 @@ /* * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. - * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp index 73134f7601e96..2df2bcd5c0a64 100644 --- a/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp +++ b/src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.hpp @@ -1,6 +1,5 @@ /* * Copyright (c) 2018, 2021, Red Hat, Inc. All rights reserved. - * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.hpp index cd61f1a50d54e..eb877921a7483 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.hpp @@ -1,6 +1,5 @@ /* * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved. - * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.hpp index 701f2362d5737..a41f2e82cb4fc 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahPassiveHeuristics.hpp @@ -1,6 +1,5 @@ /* * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved. - * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.hpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.hpp index d71d9c35820d4..35a41aff8cb93 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.hpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.hpp @@ -1,6 +1,5 @@ /* * Copyright (c) 2018, 2019, Red Hat, Inc. All rights reserved. - * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.hpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.hpp index 7a7006fdb3cb8..e39c0cbe8ac50 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.hpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahPassiveMode.hpp @@ -1,6 +1,5 @@ /* * Copyright (c) 2019, Red Hat, Inc. All rights reserved. - * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/mode/shenandoahSATBMode.cpp b/src/hotspot/share/gc/shenandoah/mode/shenandoahSATBMode.cpp index 489814fecfa60..9feff4f61b70d 100644 --- a/src/hotspot/share/gc/shenandoah/mode/shenandoahSATBMode.cpp +++ b/src/hotspot/share/gc/shenandoah/mode/shenandoahSATBMode.cpp @@ -1,6 +1,5 @@ /* * Copyright (c) 2019, 2021, Red Hat, Inc. All rights reserved. - * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp index 515a3815e3f48..0ca2c6da53907 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp @@ -1,6 +1,5 @@ /* * Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved. - * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it diff --git a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp index 625ccc9271e39..336158ca5c941 100644 --- a/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp +++ b/src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.hpp @@ -1,6 +1,5 @@ /* * Copyright (c) 2013, 2021, Red Hat, Inc. All rights reserved. - * Copyright Amazon.com Inc. or its affiliates. All Rights Reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it From 240d413dfbb093091ba0b9d85495f642c958f90b Mon Sep 17 00:00:00 2001 From: William Kemper <71722661+earthling-amzn@users.noreply.github.com> Date: Wed, 7 Jun 2023 07:57:32 -0700 Subject: [PATCH 253/254] Simplify test logic, fail if name of Shenandoah young gen pool changes (#3) --- .../mxbeans/TestChurnNotifications.java | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestChurnNotifications.java b/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestChurnNotifications.java index f8119deee771c..73c76b4d0a9b1 100644 --- a/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestChurnNotifications.java +++ b/test/hotspot/jtreg/gc/shenandoah/mxbeans/TestChurnNotifications.java @@ -113,7 +113,7 @@ * * @run main/othervm -Xmx128m -XX:+UnlockDiagnosticVMOptions -XX:+UnlockExperimentalVMOptions * -XX:+UseShenandoahGC -XX:ShenandoahGCMode=generational - * -Dprecise=false + * -Dprecise=false -Dmem.pool=Young * TestChurnNotifications */ @@ -141,16 +141,7 @@ public class TestChurnNotifications { static volatile Object sink; - private static final String DEFAULT_POOL_NAME = "Shenandoah"; - private static final String YOUNG_GEN_POOL_NAME = "Shenandoah Young Gen"; - - private static MemoryUsage getUsage(Map pools) { - MemoryUsage usage = pools.get(DEFAULT_POOL_NAME); - if (usage == null) { - usage = pools.get(YOUNG_GEN_POOL_NAME); - } - return usage; - } + private static final String POOL_NAME = "Young".equals(System.getProperty("mem.pool")) ? "Shenandoah Young Gen" : "Shenandoah"; public static void main(String[] args) throws Exception { final long startTime = System.currentTimeMillis(); @@ -165,8 +156,8 @@ public void handleNotification(Notification n, Object o) { Map mapBefore = info.getGcInfo().getMemoryUsageBeforeGc(); Map mapAfter = info.getGcInfo().getMemoryUsageAfterGc(); - MemoryUsage before = getUsage(mapBefore); - MemoryUsage after = getUsage(mapAfter); + MemoryUsage before = mapBefore.get(POOL_NAME); + MemoryUsage after = mapAfter.get(POOL_NAME); if ((before != null) && (after != null)) { long diff = before.getUsed() - after.getUsed(); From 19e62fe0f9e8371300dfd4fd02f8f11c0aa6f52a Mon Sep 17 00:00:00 2001 From: Kelvin Nilsen Date: Wed, 7 Jun 2023 17:48:32 +0000 Subject: [PATCH 254/254] Fix budgeting assertion to allow equal or greater than --- .../share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp index 5722103d73a9f..1fce4b00f720b 100644 --- a/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp +++ b/src/hotspot/share/gc/shenandoah/heuristics/shenandoahOldHeuristics.cpp @@ -79,7 +79,7 @@ bool ShenandoahOldHeuristics::prime_collection_set(ShenandoahCollectionSet* coll fragmented_available = 0; excess_fragmented_available = 0; } else { - assert(_old_generation->available() > old_evacuation_budget, "Cannot budget more than is available"); + assert(_old_generation->available() >= old_evacuation_budget, "Cannot budget more than is available"); fragmented_available = _old_generation->available() - unfragmented_available; assert(fragmented_available + unfragmented_available >= old_evacuation_budget, "Budgets do not add up"); if (fragmented_available + unfragmented_available > old_evacuation_budget) {