Skip to content

Commit 24ed3d4

Browse files
plidenpliden
committed
8230566: ZGC: Don't substitute klass pointer during array clearing
Reviewed-by: stefank, eosterlund
1 parent 7ecde2e commit 24ed3d4

16 files changed

+87
-95
lines changed

‎src/hotspot/share/gc/z/zBarrier.cpp

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ bool ZBarrier::should_mark_through(uintptr_t addr) {
7272
return true;
7373
}
7474

75-
template <bool finalizable, bool publish>
75+
template <bool follow, bool finalizable, bool publish>
7676
uintptr_t ZBarrier::mark(uintptr_t addr) {
7777
uintptr_t good_addr;
7878

@@ -89,7 +89,7 @@ uintptr_t ZBarrier::mark(uintptr_t addr) {
8989

9090
// Mark
9191
if (should_mark_through<finalizable>(addr)) {
92-
ZHeap::heap()->mark_object<finalizable, publish>(good_addr);
92+
ZHeap::heap()->mark_object<follow, finalizable, publish>(good_addr);
9393
}
9494

9595
return good_addr;
@@ -108,7 +108,7 @@ uintptr_t ZBarrier::relocate(uintptr_t addr) {
108108
}
109109

110110
uintptr_t ZBarrier::relocate_or_mark(uintptr_t addr) {
111-
return during_relocate() ? relocate(addr) : mark<Strong, Publish>(addr);
111+
return during_relocate() ? relocate(addr) : mark<Follow, Strong, Publish>(addr);
112112
}
113113

114114
uintptr_t ZBarrier::relocate_or_remap(uintptr_t addr) {
@@ -174,11 +174,11 @@ uintptr_t ZBarrier::keep_alive_barrier_on_phantom_oop_slow_path(uintptr_t addr)
174174
// Mark barrier
175175
//
176176
uintptr_t ZBarrier::mark_barrier_on_oop_slow_path(uintptr_t addr) {
177-
return mark<Strong, Overflow>(addr);
177+
return mark<Follow, Strong, Overflow>(addr);
178178
}
179179

180180
uintptr_t ZBarrier::mark_barrier_on_finalizable_oop_slow_path(uintptr_t addr) {
181-
const uintptr_t good_addr = mark<Finalizable, Overflow>(addr);
181+
const uintptr_t good_addr = mark<Follow, Finalizable, Overflow>(addr);
182182
if (ZAddress::is_good(addr)) {
183183
// If the oop was already strongly marked/good, then we do
184184
// not want to downgrade it to finalizable marked/good.
@@ -200,7 +200,15 @@ uintptr_t ZBarrier::mark_barrier_on_root_oop_slow_path(uintptr_t addr) {
200200
assert(during_mark(), "Invalid phase");
201201

202202
// Mark
203-
return mark<Strong, Publish>(addr);
203+
return mark<Follow, Strong, Publish>(addr);
204+
}
205+
206+
uintptr_t ZBarrier::mark_barrier_on_invisible_root_oop_slow_path(uintptr_t addr) {
207+
assert(SafepointSynchronize::is_at_safepoint(), "Should be at safepoint");
208+
assert(during_mark(), "Invalid phase");
209+
210+
// Mark
211+
return mark<DontFollow, Strong, Publish>(addr);
204212
}
205213

206214
//

‎src/hotspot/share/gc/z/zBarrier.hpp

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,9 @@ typedef uintptr_t (*ZBarrierSlowPath)(uintptr_t);
3232

3333
class ZBarrier : public AllStatic {
3434
private:
35+
static const bool Follow = true;
36+
static const bool DontFollow = false;
37+
3538
static const bool Strong = false;
3639
static const bool Finalizable = true;
3740

@@ -51,7 +54,7 @@ class ZBarrier : public AllStatic {
5154
static bool during_mark();
5255
static bool during_relocate();
5356
template <bool finalizable> static bool should_mark_through(uintptr_t addr);
54-
template <bool finalizable, bool publish> static uintptr_t mark(uintptr_t addr);
57+
template <bool follow, bool finalizable, bool publish> static uintptr_t mark(uintptr_t addr);
5558
static uintptr_t remap(uintptr_t addr);
5659
static uintptr_t relocate(uintptr_t addr);
5760
static uintptr_t relocate_or_mark(uintptr_t addr);
@@ -69,6 +72,7 @@ class ZBarrier : public AllStatic {
6972
static uintptr_t mark_barrier_on_oop_slow_path(uintptr_t addr);
7073
static uintptr_t mark_barrier_on_finalizable_oop_slow_path(uintptr_t addr);
7174
static uintptr_t mark_barrier_on_root_oop_slow_path(uintptr_t addr);
75+
static uintptr_t mark_barrier_on_invisible_root_oop_slow_path(uintptr_t addr);
7276

7377
static uintptr_t relocate_barrier_on_root_oop_slow_path(uintptr_t addr);
7478

@@ -106,6 +110,7 @@ class ZBarrier : public AllStatic {
106110
static void mark_barrier_on_oop_field(volatile oop* p, bool finalizable);
107111
static void mark_barrier_on_oop_array(volatile oop* p, size_t length, bool finalizable);
108112
static void mark_barrier_on_root_oop_field(oop* p);
113+
static void mark_barrier_on_invisible_root_oop_field(oop* p);
109114

110115
// Relocate barrier
111116
static void relocate_barrier_on_root_oop_field(oop* p);

‎src/hotspot/share/gc/z/zBarrier.inline.hpp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -326,6 +326,11 @@ inline void ZBarrier::mark_barrier_on_root_oop_field(oop* p) {
326326
root_barrier<is_good_or_null_fast_path, mark_barrier_on_root_oop_slow_path>(p, o);
327327
}
328328

329+
inline void ZBarrier::mark_barrier_on_invisible_root_oop_field(oop* p) {
330+
const oop o = *p;
331+
root_barrier<is_good_or_null_fast_path, mark_barrier_on_invisible_root_oop_slow_path>(p, o);
332+
}
333+
329334
//
330335
// Relocate barrier
331336
//

‎src/hotspot/share/gc/z/zHeap.hpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ class ZHeap {
140140
// Marking
141141
bool is_object_live(uintptr_t addr) const;
142142
bool is_object_strongly_live(uintptr_t addr) const;
143-
template <bool finalizable, bool publish> void mark_object(uintptr_t addr);
143+
template <bool follow, bool finalizable, bool publish> void mark_object(uintptr_t addr);
144144
void mark_start();
145145
void mark(bool initial);
146146
void mark_flush_and_free(Thread* thread);

‎src/hotspot/share/gc/z/zHeap.inline.hpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,10 +60,10 @@ inline bool ZHeap::is_object_strongly_live(uintptr_t addr) const {
6060
return page->is_object_strongly_live(addr);
6161
}
6262

63-
template <bool finalizable, bool publish>
63+
template <bool follow, bool finalizable, bool publish>
6464
inline void ZHeap::mark_object(uintptr_t addr) {
6565
assert(ZGlobalPhase == ZPhaseMark, "Mark not allowed");
66-
_mark.mark_object<finalizable, publish>(addr);
66+
_mark.mark_object<follow, finalizable, publish>(addr);
6767
}
6868

6969
inline uintptr_t ZHeap::alloc_tlab(size_t size) {

‎src/hotspot/share/gc/z/zHeapIterator.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,7 @@ void ZHeapIterator::objects_do(ObjectClosure* cl) {
193193
ZStatTimerDisable disable;
194194

195195
// Push roots to visit
196-
push_roots<ZRootsIteratorNoInvisible, false /* Concurrent */, false /* Weak */>();
196+
push_roots<ZRootsIterator, false /* Concurrent */, false /* Weak */>();
197197
push_roots<ZConcurrentRootsIteratorClaimOther, true /* Concurrent */, false /* Weak */>();
198198
if (VisitWeaks) {
199199
push_roots<ZWeakRootsIterator, false /* Concurrent */, true /* Weak */>();

‎src/hotspot/share/gc/z/zMark.cpp

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,9 @@ class ZMarkRootsIteratorClosure : public ZRootsIteratorClosure {
133133
// Update thread local address bad mask
134134
ZThreadLocalData::set_address_bad_mask(thread, ZAddressBadMask);
135135

136+
// Mark invisible root
137+
ZThreadLocalData::do_invisible_root(thread, ZBarrier::mark_barrier_on_invisible_root_oop_field);
138+
136139
// Retire TLAB
137140
ZThreadLocalAllocBuffer::retire(thread);
138141
}
@@ -156,7 +159,7 @@ class ZMarkRootsTask : public ZTask {
156159
ZMarkRootsTask(ZMark* mark) :
157160
ZTask("ZMarkRootsTask"),
158161
_mark(mark),
159-
_roots(true /* visit_invisible */, false /* visit_jvmti_weak_export */) {}
162+
_roots(false /* visit_jvmti_weak_export */) {}
160163

161164
virtual void work() {
162165
_roots.oops_do(&_cl);
@@ -339,7 +342,7 @@ void ZMark::mark_and_follow(ZMarkCache* cache, ZMarkStackEntry entry) {
339342
return;
340343
}
341344

342-
// Decode object address
345+
// Decode object address and follow flag
343346
const uintptr_t addr = entry.object_address();
344347

345348
if (!try_mark_object(cache, addr, finalizable)) {
@@ -348,7 +351,13 @@ void ZMark::mark_and_follow(ZMarkCache* cache, ZMarkStackEntry entry) {
348351
}
349352

350353
if (is_array(addr)) {
351-
follow_array_object(objArrayOop(ZOop::from_address(addr)), finalizable);
354+
// Decode follow flag
355+
const bool follow = entry.follow();
356+
357+
// The follow flag is currently only relevant for object arrays
358+
if (follow) {
359+
follow_array_object(objArrayOop(ZOop::from_address(addr)), finalizable);
360+
}
352361
} else {
353362
follow_object(ZOop::from_address(addr), finalizable);
354363
}

‎src/hotspot/share/gc/z/zMark.hpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ class ZMark {
105105

106106
bool is_initialized() const;
107107

108-
template <bool finalizable, bool publish> void mark_object(uintptr_t addr);
108+
template <bool follow, bool finalizable, bool publish> void mark_object(uintptr_t addr);
109109

110110
void start();
111111
void mark(bool initial);

‎src/hotspot/share/gc/z/zMark.inline.hpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,12 +31,12 @@
3131
#include "runtime/thread.hpp"
3232
#include "utilities/debug.hpp"
3333

34-
template <bool finalizable, bool publish>
34+
template <bool follow, bool finalizable, bool publish>
3535
inline void ZMark::mark_object(uintptr_t addr) {
3636
assert(ZAddress::is_marked(addr), "Should be marked");
3737
ZMarkThreadLocalStacks* const stacks = ZThreadLocalData::stacks(Thread::current());
3838
ZMarkStripe* const stripe = _stripes.stripe_for_addr(addr);
39-
ZMarkStackEntry entry(addr, finalizable);
39+
ZMarkStackEntry entry(addr, follow, finalizable);
4040

4141
stacks->push(&_allocator, &_stripes, stripe, entry, publish);
4242
}

‎src/hotspot/share/gc/z/zMarkStackEntry.hpp

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -35,16 +35,18 @@
3535
// ------------
3636
//
3737
// 6
38-
// 3 2 1 0
39-
// +---------------------------------------------------------------------+-+-+
40-
// |11111111 11111111 11111111 11111111 11111111 11111111 11111111 111111|1|1|
41-
// +---------------------------------------------------------------------+-+-+
42-
// | | |
43-
// | 1-1 Partial Array Flag (1-bit) * |
44-
// | |
45-
// | 0-0 Final Flag (1-bit) *
38+
// 3 3 2 1 0
39+
// +--------------------------------------------------------------------+-+-+-+
40+
// |11111111 11111111 11111111 11111111 11111111 11111111 11111111 11111|1|1|1|
41+
// +--------------------------------------------------------------------+-+-+-+
42+
// | | | |
43+
// | 2-2 Follow Flag (1-bit) * | |
44+
// | | |
45+
// | 1-1 Partial Array Flag (1-bit) * |
46+
// | |
47+
// | 0-0 Final Flag (1-bit) *
4648
// |
47-
// * 63-2 Object Address (62-bits)
49+
// * 63-3 Object Address (61-bits)
4850
//
4951
//
5052
// Partial array entry
@@ -69,7 +71,8 @@ class ZMarkStackEntry {
6971
private:
7072
typedef ZBitField<uint64_t, bool, 0, 1> field_finalizable;
7173
typedef ZBitField<uint64_t, bool, 1, 1> field_partial_array;
72-
typedef ZBitField<uint64_t, uintptr_t, 2, 62> field_object_address;
74+
typedef ZBitField<uint64_t, bool, 2, 1> field_follow;
75+
typedef ZBitField<uint64_t, uintptr_t, 3, 61> field_object_address;
7376
typedef ZBitField<uint64_t, size_t, 2, 30> field_partial_array_length;
7477
typedef ZBitField<uint64_t, size_t, 32, 32> field_partial_array_offset;
7578

@@ -83,8 +86,9 @@ class ZMarkStackEntry {
8386
// what _entry is initialized to.
8487
}
8588

86-
ZMarkStackEntry(uintptr_t object_address, bool finalizable) :
89+
ZMarkStackEntry(uintptr_t object_address, bool follow, bool finalizable) :
8790
_entry(field_object_address::encode(object_address) |
91+
field_follow::encode(follow) |
8892
field_partial_array::encode(false) |
8993
field_finalizable::encode(finalizable)) {}
9094

@@ -110,6 +114,10 @@ class ZMarkStackEntry {
110114
return field_partial_array_length::decode(_entry);
111115
}
112116

117+
bool follow() const {
118+
return field_follow::decode(_entry);
119+
}
120+
113121
uintptr_t object_address() const {
114122
return field_object_address::decode(_entry);
115123
}

0 commit comments

Comments
 (0)