Skip to content
Permalink
Browse files
8260267: ZGC: Reduce mark stack usage
Co-authored-by: Wang Chao <wchao@openjdk.org>
Co-authored-by: Hui Shi <hshi@openjdk.org>
Reviewed-by: sjohanss, ayang
  • Loading branch information
3 people committed May 20, 2021
1 parent f979523 commit aba22656829913d5f8d619a184c929a7de8431e4
@@ -26,6 +26,7 @@
#include "gc/z/zBarrier.inline.hpp"
#include "gc/z/zHeap.inline.hpp"
#include "gc/z/zOop.inline.hpp"
#include "gc/z/zThread.inline.hpp"
#include "memory/iterator.inline.hpp"
#include "oops/oop.inline.hpp"
#include "runtime/safepoint.hpp"
@@ -64,7 +65,7 @@ bool ZBarrier::should_mark_through(uintptr_t addr) {
return true;
}

template <bool follow, bool finalizable, bool publish>
template <bool gc_thread, bool follow, bool finalizable, bool publish>
uintptr_t ZBarrier::mark(uintptr_t addr) {
uintptr_t good_addr;

@@ -81,7 +82,7 @@ uintptr_t ZBarrier::mark(uintptr_t addr) {

// Mark
if (should_mark_through<finalizable>(addr)) {
ZHeap::heap()->mark_object<follow, finalizable, publish>(good_addr);
ZHeap::heap()->mark_object<gc_thread, follow, finalizable, publish>(good_addr);
}

if (finalizable) {
@@ -111,11 +112,11 @@ uintptr_t ZBarrier::relocate(uintptr_t addr) {
}

uintptr_t ZBarrier::relocate_or_mark(uintptr_t addr) {
return during_relocate() ? relocate(addr) : mark<Follow, Strong, Publish>(addr);
return during_relocate() ? relocate(addr) : mark<AnyThread, Follow, Strong, Publish>(addr);
}

uintptr_t ZBarrier::relocate_or_mark_no_follow(uintptr_t addr) {
return during_relocate() ? relocate(addr) : mark<DontFollow, Strong, Publish>(addr);
return during_relocate() ? relocate(addr) : mark<AnyThread, DontFollow, Strong, Publish>(addr);
}

uintptr_t ZBarrier::relocate_or_remap(uintptr_t addr) {
@@ -169,6 +170,13 @@ uintptr_t ZBarrier::weak_load_barrier_on_phantom_oop_slow_path(uintptr_t addr) {
//
// Keep alive barrier
//
uintptr_t ZBarrier::keep_alive_barrier_on_oop_slow_path(uintptr_t addr) {
assert(during_mark(), "Invalid phase");

// Mark
return mark<AnyThread, Follow, Strong, Overflow>(addr);
}

uintptr_t ZBarrier::keep_alive_barrier_on_weak_oop_slow_path(uintptr_t addr) {
const uintptr_t good_addr = weak_load_barrier_on_oop_slow_path(addr);
assert(ZHeap::heap()->is_object_strongly_live(good_addr), "Should be live");
@@ -186,16 +194,18 @@ uintptr_t ZBarrier::keep_alive_barrier_on_phantom_oop_slow_path(uintptr_t addr)
//
uintptr_t ZBarrier::mark_barrier_on_oop_slow_path(uintptr_t addr) {
assert(during_mark(), "Invalid phase");
assert(ZThread::is_worker(), "Invalid thread");

// Mark
return mark<Follow, Strong, Overflow>(addr);
return mark<GCThread, Follow, Strong, Overflow>(addr);
}

uintptr_t ZBarrier::mark_barrier_on_finalizable_oop_slow_path(uintptr_t addr) {
assert(during_mark(), "Invalid phase");
assert(ZThread::is_worker(), "Invalid thread");

// Mark
return mark<Follow, Finalizable, Overflow>(addr);
return mark<GCThread, Follow, Finalizable, Overflow>(addr);
}

//
@@ -33,6 +33,9 @@ typedef uintptr_t (*ZBarrierSlowPath)(uintptr_t);

class ZBarrier : public AllStatic {
private:
static const bool GCThread = true;
static const bool AnyThread = false;

static const bool Follow = true;
static const bool DontFollow = false;

@@ -55,7 +58,7 @@ class ZBarrier : public AllStatic {
static bool during_mark();
static bool during_relocate();
template <bool finalizable> static bool should_mark_through(uintptr_t addr);
template <bool follow, bool finalizable, bool publish> static uintptr_t mark(uintptr_t addr);
template <bool gc_thread, bool follow, bool finalizable, bool publish> static uintptr_t mark(uintptr_t addr);
static uintptr_t remap(uintptr_t addr);
static uintptr_t relocate(uintptr_t addr);
static uintptr_t relocate_or_mark(uintptr_t addr);
@@ -69,6 +72,7 @@ class ZBarrier : public AllStatic {
static uintptr_t weak_load_barrier_on_weak_oop_slow_path(uintptr_t addr);
static uintptr_t weak_load_barrier_on_phantom_oop_slow_path(uintptr_t addr);

static uintptr_t keep_alive_barrier_on_oop_slow_path(uintptr_t addr);
static uintptr_t keep_alive_barrier_on_weak_oop_slow_path(uintptr_t addr);
static uintptr_t keep_alive_barrier_on_phantom_oop_slow_path(uintptr_t addr);

@@ -104,10 +108,10 @@ class ZBarrier : public AllStatic {
static bool is_alive_barrier_on_phantom_oop(oop o);

// Keep alive barrier
static void keep_alive_barrier_on_oop(oop o);
static void keep_alive_barrier_on_weak_oop_field(volatile oop* p);
static void keep_alive_barrier_on_phantom_oop_field(volatile oop* p);
static void keep_alive_barrier_on_phantom_root_oop_field(oop* p);
static void keep_alive_barrier_on_oop(oop o);

// Mark barrier
static void mark_barrier_on_oop_field(volatile oop* p, bool finalizable);
@@ -364,7 +364,7 @@ inline void ZBarrier::keep_alive_barrier_on_oop(oop o) {
assert(ZAddress::is_good(addr), "Invalid address");

if (during_mark()) {
mark_barrier_on_oop_slow_path(addr);
keep_alive_barrier_on_oop_slow_path(addr);
}
}

@@ -44,6 +44,7 @@ static const ZStatPhaseCycle ZPhaseCycle("Garbage Collection Cycle");
static const ZStatPhasePause ZPhasePauseMarkStart("Pause Mark Start");
static const ZStatPhaseConcurrent ZPhaseConcurrentMark("Concurrent Mark");
static const ZStatPhaseConcurrent ZPhaseConcurrentMarkContinue("Concurrent Mark Continue");
static const ZStatPhaseConcurrent ZPhaseConcurrentMarkFree("Concurrent Mark Free");
static const ZStatPhasePause ZPhasePauseMarkEnd("Pause Mark End");
static const ZStatPhaseConcurrent ZPhaseConcurrentProcessNonStrongReferences("Concurrent Process Non-Strong References");
static const ZStatPhaseConcurrent ZPhaseConcurrentResetRelocationSet("Concurrent Reset Relocation Set");
@@ -322,6 +323,11 @@ void ZDriver::concurrent_mark_continue() {
ZHeap::heap()->mark(false /* initial */);
}

void ZDriver::concurrent_mark_free() {
ZStatTimer timer(ZPhaseConcurrentMarkFree);
ZHeap::heap()->mark_free();
}

void ZDriver::concurrent_process_non_strong_references() {
ZStatTimer timer(ZPhaseConcurrentProcessNonStrongReferences);
ZBreakpoint::at_after_reference_processing_started();
@@ -426,22 +432,25 @@ void ZDriver::gc(GCCause::Cause cause) {
concurrent(mark_continue);
}

// Phase 4: Concurrent Process Non-Strong References
// Phase 4: Concurrent Mark Free
concurrent(mark_free);

// Phase 5: Concurrent Process Non-Strong References
concurrent(process_non_strong_references);

// Phase 5: Concurrent Reset Relocation Set
// Phase 6: Concurrent Reset Relocation Set
concurrent(reset_relocation_set);

// Phase 6: Pause Verify
// Phase 7: Pause Verify
pause_verify();

// Phase 7: Concurrent Select Relocation Set
// Phase 8: Concurrent Select Relocation Set
concurrent(select_relocation_set);

// Phase 8: Pause Relocate Start
// Phase 9: Pause Relocate Start
pause_relocate_start();

// Phase 9: Concurrent Relocate
// Phase 10: Concurrent Relocate
concurrent(relocate);
}

@@ -41,6 +41,7 @@ class ZDriver : public ConcurrentGCThread {
void concurrent_mark();
bool pause_mark_end();
void concurrent_mark_continue();
void concurrent_mark_free();
void concurrent_process_non_strong_references();
void concurrent_reset_relocation_set();
void pause_verify();
@@ -284,6 +284,10 @@ bool ZHeap::mark_end() {
return true;
}

void ZHeap::mark_free() {
_mark.free();
}

void ZHeap::keep_alive(oop obj) {
ZBarrier::keep_alive_barrier_on_oop(obj);
}
@@ -122,11 +122,12 @@ class ZHeap {
// Marking
bool is_object_live(uintptr_t addr) const;
bool is_object_strongly_live(uintptr_t addr) const;
template <bool follow, bool finalizable, bool publish> void mark_object(uintptr_t addr);
template <bool gc_thread, bool follow, bool finalizable, bool publish> void mark_object(uintptr_t addr);
void mark_start();
void mark(bool initial);
void mark_flush_and_free(Thread* thread);
bool mark_end();
void mark_free();
void keep_alive(oop obj);

// Relocation set
@@ -57,10 +57,10 @@ inline bool ZHeap::is_object_strongly_live(uintptr_t addr) const {
return page->is_object_strongly_live(addr);
}

template <bool follow, bool finalizable, bool publish>
template <bool gc_thread, bool follow, bool finalizable, bool publish>
inline void ZHeap::mark_object(uintptr_t addr) {
assert(ZGlobalPhase == ZPhaseMark, "Mark not allowed");
_mark.mark_object<follow, finalizable, publish>(addr);
_mark.mark_object<gc_thread, follow, finalizable, publish>(addr);
}

inline uintptr_t ZHeap::alloc_tlab(size_t size) {
@@ -278,28 +278,6 @@ void ZMark::follow_object(oop obj, bool finalizable) {
}
}

bool ZMark::try_mark_object(ZMarkCache* cache, uintptr_t addr, bool finalizable) {
ZPage* const page = _page_table->get(addr);
if (page->is_allocating()) {
// Newly allocated objects are implicitly marked
return false;
}

// Try mark object
bool inc_live = false;
const bool success = page->mark_object(addr, finalizable, inc_live);
if (inc_live) {
// Update live objects/bytes for page. We use the aligned object
// size since that is the actual number of bytes used on the page
// and alignment paddings can never be reclaimed.
const size_t size = ZUtils::object_size(addr);
const size_t aligned_size = align_up(size, page->object_alignment());
cache->inc_live(page, aligned_size);
}

return success;
}

void ZMark::mark_and_follow(ZMarkCache* cache, ZMarkStackEntry entry) {
// Decode flags
const bool finalizable = entry.finalizable();
@@ -310,24 +288,38 @@ void ZMark::mark_and_follow(ZMarkCache* cache, ZMarkStackEntry entry) {
return;
}

// Decode object address and follow flag
// Decode object address and additional flags
const uintptr_t addr = entry.object_address();
const bool mark = entry.mark();
bool inc_live = entry.inc_live();
const bool follow = entry.follow();

if (!try_mark_object(cache, addr, finalizable)) {
ZPage* const page = _page_table->get(addr);
assert(page->is_relocatable(), "Invalid page state");

// Mark
if (mark && !page->mark_object(addr, finalizable, inc_live)) {
// Already marked
return;
}

if (is_array(addr)) {
// Decode follow flag
const bool follow = entry.follow();
// Increment live
if (inc_live) {
// Update live objects/bytes for page. We use the aligned object
// size since that is the actual number of bytes used on the page
// and alignment paddings can never be reclaimed.
const size_t size = ZUtils::object_size(addr);
const size_t aligned_size = align_up(size, page->object_alignment());
cache->inc_live(page, aligned_size);
}

// The follow flag is currently only relevant for object arrays
if (follow) {
// Follow
if (follow) {
if (is_array(addr)) {
follow_array_object(objArrayOop(ZOop::from_address(addr)), finalizable);
} else {
follow_object(ZOop::from_address(addr), finalizable);
}
} else {
follow_object(ZOop::from_address(addr), finalizable);
}
}

@@ -776,6 +768,14 @@ bool ZMark::end() {
return true;
}

void ZMark::free() {
// Free any unused mark stack space
_allocator.free();

// Update statistics
ZStatMark::set_at_mark_free(_allocator.size());
}

void ZMark::flush_and_free() {
Thread* const thread = Thread::current();
flush_and_free(thread);
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -64,7 +64,6 @@ class ZMark {
void follow_partial_array(ZMarkStackEntry entry, bool finalizable);
void follow_array_object(objArrayOop obj, bool finalizable);
void follow_object(oop obj, bool finalizable);
bool try_mark_object(ZMarkCache* cache, uintptr_t addr, bool finalizable);
void mark_and_follow(ZMarkCache* cache, ZMarkStackEntry entry);

template <typename T> bool drain(ZMarkStripe* stripe,
@@ -101,11 +100,12 @@ class ZMark {

bool is_initialized() const;

template <bool follow, bool finalizable, bool publish> void mark_object(uintptr_t addr);
template <bool gc_thread, bool follow, bool finalizable, bool publish> void mark_object(uintptr_t addr);

void start();
void mark(bool initial);
bool end();
void free();

void flush_and_free();
bool flush_and_free(Thread* thread);

0 comments on commit aba2265

Please sign in to comment.